2024-11-11 03:39:37,857 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 03:39:37,873 main DEBUG Took 0.013798 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 03:39:37,874 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 03:39:37,874 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 03:39:37,875 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 03:39:37,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,887 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 03:39:37,907 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,910 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,911 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,913 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,914 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,916 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,918 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,919 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,920 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,921 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,923 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,924 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,925 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:39:37,925 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,926 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 03:39:37,928 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:39:37,930 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 03:39:37,933 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 03:39:37,934 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 03:39:37,936 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 03:39:37,937 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 03:39:37,949 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 03:39:37,953 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 03:39:37,956 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 03:39:37,957 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 03:39:37,957 main DEBUG createAppenders(={Console}) 2024-11-11 03:39:37,959 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 03:39:37,959 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 03:39:37,960 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 03:39:37,961 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 03:39:37,961 main DEBUG OutputStream closed 2024-11-11 03:39:37,962 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 03:39:37,962 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 03:39:37,963 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 03:39:38,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 03:39:38,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 03:39:38,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 03:39:38,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 03:39:38,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 03:39:38,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 03:39:38,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 03:39:38,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 03:39:38,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 03:39:38,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 03:39:38,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 03:39:38,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 03:39:38,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 03:39:38,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 03:39:38,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 03:39:38,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 03:39:38,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 03:39:38,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 03:39:38,081 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 03:39:38,082 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 03:39:38,082 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 03:39:38,083 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T03:39:38,332 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d 2024-11-11 03:39:38,335 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 03:39:38,336 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T03:39:38,344 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T03:39:38,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T03:39:38,391 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5, deleteOnExit=true 2024-11-11T03:39:38,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T03:39:38,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/test.cache.data in system properties and HBase conf 2024-11-11T03:39:38,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T03:39:38,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/hadoop.log.dir in system properties and HBase conf 2024-11-11T03:39:38,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T03:39:38,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T03:39:38,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T03:39:38,513 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T03:39:38,650 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T03:39:38,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T03:39:38,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T03:39:38,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T03:39:38,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T03:39:38,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T03:39:38,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T03:39:38,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T03:39:38,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T03:39:38,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T03:39:38,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/nfs.dump.dir in system properties and HBase conf 2024-11-11T03:39:38,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/java.io.tmpdir in system properties and HBase conf 2024-11-11T03:39:38,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T03:39:38,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T03:39:38,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T03:39:39,958 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T03:39:40,066 INFO [Time-limited test {}] log.Log(170): Logging initialized @3043ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T03:39:40,181 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T03:39:40,271 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T03:39:40,307 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T03:39:40,307 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T03:39:40,309 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T03:39:40,327 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T03:39:40,330 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/hadoop.log.dir/,AVAILABLE} 2024-11-11T03:39:40,331 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T03:39:40,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/java.io.tmpdir/jetty-localhost-42795-hadoop-hdfs-3_4_1-tests_jar-_-any-2577601116613708111/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T03:39:40,526 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42795} 2024-11-11T03:39:40,526 INFO [Time-limited test {}] server.Server(415): Started @3505ms 2024-11-11T03:39:41,113 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T03:39:41,122 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T03:39:41,123 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T03:39:41,123 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T03:39:41,124 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T03:39:41,124 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/hadoop.log.dir/,AVAILABLE} 2024-11-11T03:39:41,125 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T03:39:41,247 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/java.io.tmpdir/jetty-localhost-44403-hadoop-hdfs-3_4_1-tests_jar-_-any-415317325040814922/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T03:39:41,247 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:44403} 2024-11-11T03:39:41,247 INFO [Time-limited test {}] server.Server(415): Started @4226ms 2024-11-11T03:39:41,301 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T03:39:42,499 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/data/data1/current/BP-220414605-172.17.0.2-1731296379415/current, will proceed with Du for space computation calculation, 2024-11-11T03:39:42,499 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/data/data2/current/BP-220414605-172.17.0.2-1731296379415/current, will proceed with Du for space computation calculation, 2024-11-11T03:39:42,532 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T03:39:42,577 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5fda012ab1234096 with lease ID 0x9f3f3f6e2e249877: Processing first storage report for DS-2828dd91-3a0c-42cd-b72c-260af934d204 from datanode DatanodeRegistration(127.0.0.1:36307, datanodeUuid=0c3298f7-bd01-4a84-915f-4c7cc85b3801, infoPort=44717, infoSecurePort=0, ipcPort=46127, storageInfo=lv=-57;cid=testClusterID;nsid=538153718;c=1731296379415) 2024-11-11T03:39:42,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5fda012ab1234096 with lease ID 0x9f3f3f6e2e249877: from storage DS-2828dd91-3a0c-42cd-b72c-260af934d204 node DatanodeRegistration(127.0.0.1:36307, datanodeUuid=0c3298f7-bd01-4a84-915f-4c7cc85b3801, infoPort=44717, infoSecurePort=0, ipcPort=46127, storageInfo=lv=-57;cid=testClusterID;nsid=538153718;c=1731296379415), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-11T03:39:42,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5fda012ab1234096 with lease ID 0x9f3f3f6e2e249877: Processing first storage report for DS-718f1a08-060a-4a8a-9cc0-b5a7ce73a97a from datanode DatanodeRegistration(127.0.0.1:36307, datanodeUuid=0c3298f7-bd01-4a84-915f-4c7cc85b3801, infoPort=44717, infoSecurePort=0, ipcPort=46127, storageInfo=lv=-57;cid=testClusterID;nsid=538153718;c=1731296379415) 2024-11-11T03:39:42,580 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5fda012ab1234096 with lease ID 0x9f3f3f6e2e249877: from storage DS-718f1a08-060a-4a8a-9cc0-b5a7ce73a97a node DatanodeRegistration(127.0.0.1:36307, datanodeUuid=0c3298f7-bd01-4a84-915f-4c7cc85b3801, infoPort=44717, infoSecurePort=0, ipcPort=46127, storageInfo=lv=-57;cid=testClusterID;nsid=538153718;c=1731296379415), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T03:39:42,624 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d 2024-11-11T03:39:42,690 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/zookeeper_0, clientPort=50016, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T03:39:42,699 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50016 2024-11-11T03:39:42,708 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:39:42,711 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:39:42,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741825_1001 (size=7) 2024-11-11T03:39:43,314 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c with version=8 2024-11-11T03:39:43,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/hbase-staging 2024-11-11T03:39:43,403 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T03:39:43,673 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3ffe7f9d089e:0 server-side Connection retries=45 2024-11-11T03:39:43,686 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:39:43,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T03:39:43,692 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T03:39:43,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:39:43,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T03:39:43,833 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T03:39:43,893 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T03:39:43,902 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T03:39:43,906 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T03:39:43,929 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33006 (auto-detected) 2024-11-11T03:39:43,930 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T03:39:43,948 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45295 2024-11-11T03:39:43,976 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45295 connecting to ZooKeeper ensemble=127.0.0.1:50016 2024-11-11T03:39:44,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:452950x0, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:39:44,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45295-0x1012943e1960000 connected 2024-11-11T03:39:44,152 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:39:44,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:39:44,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:39:44,167 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c, hbase.cluster.distributed=false 2024-11-11T03:39:44,197 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T03:39:44,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45295 2024-11-11T03:39:44,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45295 2024-11-11T03:39:44,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45295 2024-11-11T03:39:44,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45295 2024-11-11T03:39:44,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45295 2024-11-11T03:39:44,325 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3ffe7f9d089e:0 server-side Connection retries=45 2024-11-11T03:39:44,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:39:44,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T03:39:44,327 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T03:39:44,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:39:44,327 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T03:39:44,331 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T03:39:44,335 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T03:39:44,336 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34473 2024-11-11T03:39:44,339 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34473 connecting to ZooKeeper ensemble=127.0.0.1:50016 2024-11-11T03:39:44,341 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:39:44,346 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:39:44,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344730x0, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:39:44,368 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34473-0x1012943e1960001 connected 2024-11-11T03:39:44,368 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:39:44,372 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T03:39:44,381 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T03:39:44,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T03:39:44,389 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T03:39:44,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34473 2024-11-11T03:39:44,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34473 2024-11-11T03:39:44,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34473 2024-11-11T03:39:44,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34473 2024-11-11T03:39:44,393 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34473 2024-11-11T03:39:44,408 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3ffe7f9d089e:45295 2024-11-11T03:39:44,409 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:44,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:39:44,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:39:44,428 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:44,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T03:39:44,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:44,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:44,459 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T03:39:44,461 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3ffe7f9d089e,45295,1731296383499 from backup master directory 2024-11-11T03:39:44,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:39:44,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:44,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:39:44,472 WARN [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T03:39:44,472 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:44,474 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T03:39:44,476 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T03:39:44,537 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/hbase.id] with ID: f38a4edf-cf51-45c3-8d97-e3140b469b36 2024-11-11T03:39:44,537 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/.tmp/hbase.id 2024-11-11T03:39:44,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741826_1002 (size=42) 2024-11-11T03:39:44,548 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/.tmp/hbase.id]:[hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/hbase.id] 2024-11-11T03:39:44,600 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:39:44,605 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T03:39:44,624 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-11T03:39:44,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:44,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:44,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741827_1003 (size=196) 2024-11-11T03:39:44,692 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:39:44,696 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T03:39:44,715 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:44,720 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T03:39:44,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741828_1004 (size=1189) 2024-11-11T03:39:45,170 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store 2024-11-11T03:39:45,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741829_1005 (size=34) 2024-11-11T03:39:45,201 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T03:39:45,205 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:39:45,207 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T03:39:45,207 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:39:45,207 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:39:45,209 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T03:39:45,210 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:39:45,210 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:39:45,212 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731296385206Disabling compacts and flushes for region at 1731296385206Disabling writes for close at 1731296385210 (+4 ms)Writing region close event to WAL at 1731296385210Closed at 1731296385210 2024-11-11T03:39:45,214 WARN [master/3ffe7f9d089e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/.initializing 2024-11-11T03:39:45,214 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/WALs/3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:45,225 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T03:39:45,246 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3ffe7f9d089e%2C45295%2C1731296383499, suffix=, logDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/WALs/3ffe7f9d089e,45295,1731296383499, archiveDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/oldWALs, maxLogs=10 2024-11-11T03:39:45,278 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/WALs/3ffe7f9d089e,45295,1731296383499/3ffe7f9d089e%2C45295%2C1731296383499.1731296385253, exclude list is [], retry=0 2024-11-11T03:39:45,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-2828dd91-3a0c-42cd-b72c-260af934d204,DISK] 2024-11-11T03:39:45,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T03:39:45,347 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/WALs/3ffe7f9d089e,45295,1731296383499/3ffe7f9d089e%2C45295%2C1731296383499.1731296385253 2024-11-11T03:39:45,349 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44717:44717)] 2024-11-11T03:39:45,349 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:39:45,349 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:39:45,352 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,353 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T03:39:45,429 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:45,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:45,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T03:39:45,438 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:45,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:39:45,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T03:39:45,445 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:45,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:39:45,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T03:39:45,450 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:45,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:39:45,452 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,457 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,458 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,464 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,464 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,468 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T03:39:45,472 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:39:45,477 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:39:45,479 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63900898, jitterRate=-0.04780241847038269}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T03:39:45,486 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731296385366Initializing all the Stores at 1731296385368 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731296385369 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296385370 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296385370Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296385370Cleaning up temporary data from old regions at 1731296385464 (+94 ms)Region opened successfully at 1731296385486 (+22 ms) 2024-11-11T03:39:45,488 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T03:39:45,522 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a02b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3ffe7f9d089e/172.17.0.2:0 2024-11-11T03:39:45,550 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T03:39:45,564 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T03:39:45,564 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T03:39:45,567 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T03:39:45,569 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T03:39:45,574 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T03:39:45,574 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T03:39:45,607 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T03:39:45,618 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T03:39:45,641 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T03:39:45,644 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T03:39:45,646 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T03:39:45,658 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T03:39:45,661 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T03:39:45,666 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T03:39:45,675 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T03:39:45,677 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T03:39:45,687 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T03:39:45,708 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T03:39:45,720 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T03:39:45,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T03:39:45,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T03:39:45,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:45,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:45,737 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3ffe7f9d089e,45295,1731296383499, sessionid=0x1012943e1960000, setting cluster-up flag (Was=false) 2024-11-11T03:39:45,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:45,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:45,808 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T03:39:45,810 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:45,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:45,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:45,862 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T03:39:45,867 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:45,877 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T03:39:45,898 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(746): ClusterId : f38a4edf-cf51-45c3-8d97-e3140b469b36 2024-11-11T03:39:45,900 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T03:39:45,914 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T03:39:45,914 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T03:39:45,926 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T03:39:45,926 DEBUG [RS:0;3ffe7f9d089e:34473 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44759f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3ffe7f9d089e/172.17.0.2:0 2024-11-11T03:39:45,942 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3ffe7f9d089e:34473 2024-11-11T03:39:45,945 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T03:39:45,944 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T03:39:45,945 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T03:39:45,945 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T03:39:45,947 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(2659): reportForDuty to master=3ffe7f9d089e,45295,1731296383499 with port=34473, startcode=1731296384288 2024-11-11T03:39:45,953 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T03:39:45,958 DEBUG [RS:0;3ffe7f9d089e:34473 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T03:39:45,959 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T03:39:45,964 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3ffe7f9d089e,45295,1731296383499 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T03:39:45,971 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3ffe7f9d089e:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:39:45,971 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3ffe7f9d089e:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:39:45,971 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3ffe7f9d089e:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:39:45,971 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3ffe7f9d089e:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:39:45,972 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3ffe7f9d089e:0, corePoolSize=10, maxPoolSize=10 2024-11-11T03:39:45,972 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:45,972 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3ffe7f9d089e:0, corePoolSize=2, maxPoolSize=2 2024-11-11T03:39:45,972 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:45,986 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T03:39:45,987 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T03:39:45,996 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731296415996 2024-11-11T03:39:45,997 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41255, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T03:39:45,998 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T03:39:45,999 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:45,999 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T03:39:45,999 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T03:39:46,003 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T03:39:46,004 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T03:39:46,004 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T03:39:46,004 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45295 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:46,004 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T03:39:46,006 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,015 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T03:39:46,017 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T03:39:46,017 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T03:39:46,027 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T03:39:46,027 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T03:39:46,030 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3ffe7f9d089e:0:becomeActiveMaster-HFileCleaner.large.0-1731296386029,5,FailOnTimeoutGroup] 2024-11-11T03:39:46,032 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3ffe7f9d089e:0:becomeActiveMaster-HFileCleaner.small.0-1731296386031,5,FailOnTimeoutGroup] 2024-11-11T03:39:46,032 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,032 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T03:39:46,034 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,034 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,040 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T03:39:46,040 WARN [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T03:39:46,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741831_1007 (size=1321) 2024-11-11T03:39:46,141 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(2659): reportForDuty to master=3ffe7f9d089e,45295,1731296383499 with port=34473, startcode=1731296384288 2024-11-11T03:39:46,144 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45295 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:46,146 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45295 {}] master.ServerManager(517): Registering regionserver=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:46,154 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:39:46,154 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44167 2024-11-11T03:39:46,154 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T03:39:46,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T03:39:46,221 DEBUG [RS:0;3ffe7f9d089e:34473 {}] zookeeper.ZKUtil(111): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:46,221 WARN [RS:0;3ffe7f9d089e:34473 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T03:39:46,221 INFO [RS:0;3ffe7f9d089e:34473 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T03:39:46,222 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/WALs/3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:46,224 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3ffe7f9d089e,34473,1731296384288] 2024-11-11T03:39:46,248 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T03:39:46,265 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T03:39:46,270 INFO [RS:0;3ffe7f9d089e:34473 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T03:39:46,270 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,272 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T03:39:46,278 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T03:39:46,280 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,280 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,280 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,281 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,281 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,281 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,281 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3ffe7f9d089e:0, corePoolSize=2, maxPoolSize=2 2024-11-11T03:39:46,281 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,282 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,282 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,282 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,282 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,282 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3ffe7f9d089e:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:39:46,282 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3ffe7f9d089e:0, corePoolSize=3, maxPoolSize=3 2024-11-11T03:39:46,283 DEBUG [RS:0;3ffe7f9d089e:34473 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0, corePoolSize=3, maxPoolSize=3 2024-11-11T03:39:46,284 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,284 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,284 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,284 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,285 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,285 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,34473,1731296384288-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T03:39:46,306 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T03:39:46,308 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,34473,1731296384288-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,308 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,308 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.Replication(171): 3ffe7f9d089e,34473,1731296384288 started 2024-11-11T03:39:46,328 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:46,329 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1482): Serving as 3ffe7f9d089e,34473,1731296384288, RpcServer on 3ffe7f9d089e/172.17.0.2:34473, sessionid=0x1012943e1960001 2024-11-11T03:39:46,330 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T03:39:46,330 DEBUG [RS:0;3ffe7f9d089e:34473 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:46,330 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3ffe7f9d089e,34473,1731296384288' 2024-11-11T03:39:46,330 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T03:39:46,331 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T03:39:46,333 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T03:39:46,333 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T03:39:46,333 DEBUG [RS:0;3ffe7f9d089e:34473 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:46,333 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3ffe7f9d089e,34473,1731296384288' 2024-11-11T03:39:46,333 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T03:39:46,334 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T03:39:46,335 DEBUG [RS:0;3ffe7f9d089e:34473 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T03:39:46,335 INFO [RS:0;3ffe7f9d089e:34473 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T03:39:46,335 INFO [RS:0;3ffe7f9d089e:34473 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T03:39:46,441 INFO [RS:0;3ffe7f9d089e:34473 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T03:39:46,444 INFO [RS:0;3ffe7f9d089e:34473 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3ffe7f9d089e%2C34473%2C1731296384288, suffix=, logDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/WALs/3ffe7f9d089e,34473,1731296384288, archiveDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/oldWALs, maxLogs=32 2024-11-11T03:39:46,447 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T03:39:46,448 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:39:46,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741832_1008 (size=32) 2024-11-11T03:39:46,469 DEBUG [RS:0;3ffe7f9d089e:34473 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/WALs/3ffe7f9d089e,34473,1731296384288/3ffe7f9d089e%2C34473%2C1731296384288.1731296386448, exclude list is [], retry=0 2024-11-11T03:39:46,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-2828dd91-3a0c-42cd-b72c-260af934d204,DISK] 2024-11-11T03:39:46,478 INFO [RS:0;3ffe7f9d089e:34473 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/WALs/3ffe7f9d089e,34473,1731296384288/3ffe7f9d089e%2C34473%2C1731296384288.1731296386448 2024-11-11T03:39:46,483 DEBUG [RS:0;3ffe7f9d089e:34473 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44717:44717)] 2024-11-11T03:39:46,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:39:46,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T03:39:46,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T03:39:46,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:46,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:46,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T03:39:46,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T03:39:46,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:46,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:46,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T03:39:46,873 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T03:39:46,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:46,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:46,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T03:39:46,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T03:39:46,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:46,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:46,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T03:39:46,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740 2024-11-11T03:39:46,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740 2024-11-11T03:39:46,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T03:39:46,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T03:39:46,883 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:39:46,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T03:39:46,889 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:39:46,890 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58894791, jitterRate=-0.1223992258310318}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:39:46,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731296386860Initializing all the Stores at 1731296386862 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731296386862Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731296386862Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296386862Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731296386862Cleaning up temporary data from old regions at 1731296386883 (+21 ms)Region opened successfully at 1731296386892 (+9 ms) 2024-11-11T03:39:46,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T03:39:46,893 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T03:39:46,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T03:39:46,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T03:39:46,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T03:39:46,894 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T03:39:46,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731296386893Disabling compacts and flushes for region at 1731296386893Disabling writes for close at 1731296386893Writing region close event to WAL at 1731296386894 (+1 ms)Closed at 1731296386894 2024-11-11T03:39:46,897 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T03:39:46,897 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T03:39:46,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T03:39:46,910 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T03:39:46,912 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T03:39:47,068 DEBUG [3ffe7f9d089e:45295 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T03:39:47,082 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:47,087 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3ffe7f9d089e,34473,1731296384288, state=OPENING 2024-11-11T03:39:47,132 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T03:39:47,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:47,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:39:47,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:39:47,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:39:47,144 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T03:39:47,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:39:47,325 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T03:39:47,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T03:39:47,338 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T03:39:47,339 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T03:39:47,339 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T03:39:47,343 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3ffe7f9d089e%2C34473%2C1731296384288.meta, suffix=.meta, logDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/WALs/3ffe7f9d089e,34473,1731296384288, archiveDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/oldWALs, maxLogs=32 2024-11-11T03:39:47,358 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/WALs/3ffe7f9d089e,34473,1731296384288/3ffe7f9d089e%2C34473%2C1731296384288.meta.1731296387345.meta, exclude list is [], retry=0 2024-11-11T03:39:47,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-2828dd91-3a0c-42cd-b72c-260af934d204,DISK] 2024-11-11T03:39:47,364 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/WALs/3ffe7f9d089e,34473,1731296384288/3ffe7f9d089e%2C34473%2C1731296384288.meta.1731296387345.meta 2024-11-11T03:39:47,365 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44717:44717)] 2024-11-11T03:39:47,365 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:39:47,367 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T03:39:47,369 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T03:39:47,373 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T03:39:47,376 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T03:39:47,377 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:39:47,377 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T03:39:47,377 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T03:39:47,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T03:39:47,382 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T03:39:47,382 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:47,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:47,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T03:39:47,384 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T03:39:47,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:47,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:47,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T03:39:47,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T03:39:47,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:47,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:47,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T03:39:47,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T03:39:47,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:47,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:39:47,390 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T03:39:47,392 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740 2024-11-11T03:39:47,394 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740 2024-11-11T03:39:47,397 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T03:39:47,397 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T03:39:47,398 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:39:47,401 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T03:39:47,402 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61579369, jitterRate=-0.08239589631557465}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:39:47,402 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T03:39:47,404 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731296387378Writing region info on filesystem at 1731296387378Initializing all the Stores at 1731296387380 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731296387380Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731296387380Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296387380Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731296387380Cleaning up temporary data from old regions at 1731296387397 (+17 ms)Running coprocessor post-open hooks at 1731296387403 (+6 ms)Region opened successfully at 1731296387404 (+1 ms) 2024-11-11T03:39:47,410 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731296387318 2024-11-11T03:39:47,421 DEBUG [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T03:39:47,421 INFO [RS_OPEN_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T03:39:47,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:47,425 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3ffe7f9d089e,34473,1731296384288, state=OPEN 2024-11-11T03:39:47,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T03:39:47,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T03:39:47,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:39:47,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:39:47,468 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:47,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T03:39:47,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3ffe7f9d089e,34473,1731296384288 in 323 msec 2024-11-11T03:39:47,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T03:39:47,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 576 msec 2024-11-11T03:39:47,486 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T03:39:47,486 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T03:39:47,503 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:47,504 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:47,522 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:47,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52061, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:47,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6390 sec 2024-11-11T03:39:47,544 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731296387544, completionTime=-1 2024-11-11T03:39:47,547 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T03:39:47,547 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T03:39:47,571 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T03:39:47,571 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731296447571 2024-11-11T03:39:47,571 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731296507571 2024-11-11T03:39:47,571 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-11T03:39:47,573 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,45295,1731296383499-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:47,574 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,45295,1731296383499-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:47,574 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,45295,1731296383499-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:47,575 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3ffe7f9d089e:45295, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:47,575 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:47,576 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:47,581 DEBUG [master/3ffe7f9d089e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T03:39:47,600 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.128sec 2024-11-11T03:39:47,601 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T03:39:47,602 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T03:39:47,603 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T03:39:47,604 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T03:39:47,604 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T03:39:47,605 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,45295,1731296383499-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T03:39:47,605 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,45295,1731296383499-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T03:39:47,614 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T03:39:47,614 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T03:39:47,615 INFO [master/3ffe7f9d089e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ffe7f9d089e,45295,1731296383499-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:39:47,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:47,713 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T03:39:47,713 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T03:39:47,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:47,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:47,725 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5389b0e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:47,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:47,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:47,738 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:47,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:47,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:47,780 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=147, ProcessCount=11, AvailableMemoryMB=12892 2024-11-11T03:39:47,788 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T03:39:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a30b8b7 2024-11-11T03:39:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:39:47,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:39:47,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:39:47,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:39:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T03:39:47,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T03:39:47,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T03:39:47,816 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:47,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T03:39:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T03:39:47,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741835_1011 (size=1061) 2024-11-11T03:39:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T03:39:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T03:39:48,239 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:39:48,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741836_1012 (size=53) 2024-11-11T03:39:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T03:39:48,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:39:48,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 488bb5013d196b74e2d5128b2d5b7358, disabling compactions & flushes 2024-11-11T03:39:48,650 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:48,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:48,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. after waiting 0 ms 2024-11-11T03:39:48,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:48,651 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:48,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 488bb5013d196b74e2d5128b2d5b7358: Waiting for close lock at 1731296388650Disabling compacts and flushes for region at 1731296388650Disabling writes for close at 1731296388650Writing region close event to WAL at 1731296388651 (+1 ms)Closed at 1731296388651 2024-11-11T03:39:48,656 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T03:39:48,663 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731296388657"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731296388657"}]},"ts":"1731296388657"} 2024-11-11T03:39:48,667 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T03:39:48,669 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T03:39:48,671 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296388669"}]},"ts":"1731296388669"} 2024-11-11T03:39:48,676 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T03:39:48,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, ASSIGN}] 2024-11-11T03:39:48,680 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, ASSIGN 2024-11-11T03:39:48,682 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, ASSIGN; state=OFFLINE, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=false 2024-11-11T03:39:48,834 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=488bb5013d196b74e2d5128b2d5b7358, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:48,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, ASSIGN because future has completed 2024-11-11T03:39:48,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:39:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T03:39:49,006 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:49,007 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:39:49,007 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,007 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:39:49,007 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,007 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,010 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,013 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:39:49,013 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 488bb5013d196b74e2d5128b2d5b7358 columnFamilyName A 2024-11-11T03:39:49,014 DEBUG [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:49,015 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.HStore(327): Store=488bb5013d196b74e2d5128b2d5b7358/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:39:49,015 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,017 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:39:49,017 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 488bb5013d196b74e2d5128b2d5b7358 columnFamilyName B 2024-11-11T03:39:49,018 DEBUG [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:49,018 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.HStore(327): Store=488bb5013d196b74e2d5128b2d5b7358/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:39:49,019 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,020 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:39:49,021 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 488bb5013d196b74e2d5128b2d5b7358 columnFamilyName C 2024-11-11T03:39:49,021 DEBUG [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:39:49,022 INFO [StoreOpener-488bb5013d196b74e2d5128b2d5b7358-1 {}] regionserver.HStore(327): Store=488bb5013d196b74e2d5128b2d5b7358/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:39:49,022 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:49,022 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,024 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,025 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,027 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,027 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,028 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:39:49,030 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,033 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:39:49,035 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 488bb5013d196b74e2d5128b2d5b7358; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70512880, jitterRate=0.05072379112243652}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:39:49,035 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:49,037 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 488bb5013d196b74e2d5128b2d5b7358: Running coprocessor pre-open hook at 1731296389007Writing region info on filesystem at 1731296389008 (+1 ms)Initializing all the Stores at 1731296389009 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296389009Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296389009Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296389009Cleaning up temporary data from old regions at 1731296389027 (+18 ms)Running coprocessor post-open hooks at 1731296389035 (+8 ms)Region opened successfully at 1731296389036 (+1 ms) 2024-11-11T03:39:49,039 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., pid=6, masterSystemTime=1731296388994 2024-11-11T03:39:49,042 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:49,043 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:49,044 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=488bb5013d196b74e2d5128b2d5b7358, regionState=OPEN, openSeqNum=2, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:49,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:39:49,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T03:39:49,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 in 210 msec 2024-11-11T03:39:49,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T03:39:49,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, ASSIGN in 378 msec 2024-11-11T03:39:49,063 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T03:39:49,063 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296389063"}]},"ts":"1731296389063"} 2024-11-11T03:39:49,067 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T03:39:49,069 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T03:39:49,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2630 sec 2024-11-11T03:39:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T03:39:49,974 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:39:49,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:49,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:49,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:49,983 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dee243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:49,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5efef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:49,989 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:49,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:49,996 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T03:39:49,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ffe7f9d089e,45295,1731296383499 2024-11-11T03:39:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2779277 2024-11-11T03:39:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:39:50,003 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:39:50,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,009 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,021 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,025 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c7aae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,043 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,048 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763c2183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd1a9dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,059 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceb793a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,062 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60658e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643444eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,069 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a18c602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4085c543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,090 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c44f32e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,093 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306cf84f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:39:50,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:39:50,097 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:39:50,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:39:50,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:39:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c17951c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:39:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:39:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:39:50,100 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:39:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315b49bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:39:50,115 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,115 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,116 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,116 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,116 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,122 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,122 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,127 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,127 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:39:50,128 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,133 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:39:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:39:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:39:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T03:39:50,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:39:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T03:39:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,144 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:39:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:39:50,152 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:39:50,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:39:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:50,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:39:50,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:50,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:50,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:50,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:50,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:50,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T03:39:50,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T03:39:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296450314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296450323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296450327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296450333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296450341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/603c522a70c0454f980b567bae66a7b9 is 50, key is test_row_0/A:col10/1731296390205/Put/seqid=0 2024-11-11T03:39:50,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741837_1013 (size=14341) 2024-11-11T03:39:50,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/603c522a70c0454f980b567bae66a7b9 2024-11-11T03:39:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T03:39:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296450464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296450465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296450465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296450473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/0910b28ff7dc4c229d00754540d764e8 is 50, key is test_row_0/B:col10/1731296390205/Put/seqid=0 2024-11-11T03:39:50,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296450465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T03:39:50,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:50,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741838_1014 (size=12001) 2024-11-11T03:39:50,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T03:39:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296450683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296450684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296450693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296450685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296450693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T03:39:50,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T03:39:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/0910b28ff7dc4c229d00754540d764e8 2024-11-11T03:39:50,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f1369c0b92a443e5aedd91dc7b137549 is 50, key is test_row_0/C:col10/1731296390205/Put/seqid=0 2024-11-11T03:39:50,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T03:39:50,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:50,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:50,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:51,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741839_1015 (size=12001) 2024-11-11T03:39:51,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f1369c0b92a443e5aedd91dc7b137549 2024-11-11T03:39:51,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296451004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296451004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296451004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296451012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296451013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/603c522a70c0454f980b567bae66a7b9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/603c522a70c0454f980b567bae66a7b9 2024-11-11T03:39:51,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/603c522a70c0454f980b567bae66a7b9, entries=200, sequenceid=13, filesize=14.0 K 2024-11-11T03:39:51,049 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T03:39:51,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/0910b28ff7dc4c229d00754540d764e8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/0910b28ff7dc4c229d00754540d764e8 2024-11-11T03:39:51,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/0910b28ff7dc4c229d00754540d764e8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T03:39:51,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f1369c0b92a443e5aedd91dc7b137549 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1369c0b92a443e5aedd91dc7b137549 2024-11-11T03:39:51,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1369c0b92a443e5aedd91dc7b137549, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T03:39:51,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 488bb5013d196b74e2d5128b2d5b7358 in 891ms, sequenceid=13, compaction requested=false 2024-11-11T03:39:51,109 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T03:39:51,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:51,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T03:39:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:51,152 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T03:39:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:51,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:51,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/a28329239071493d82e698a23888d235 is 50, key is test_row_0/A:col10/1731296390331/Put/seqid=0 2024-11-11T03:39:51,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741840_1016 (size=12001) 2024-11-11T03:39:51,211 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/a28329239071493d82e698a23888d235 2024-11-11T03:39:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/61bfb881ee374328a668c4fee4d3d72d is 50, key is test_row_0/B:col10/1731296390331/Put/seqid=0 2024-11-11T03:39:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T03:39:51,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741841_1017 (size=12001) 2024-11-11T03:39:51,307 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/61bfb881ee374328a668c4fee4d3d72d 2024-11-11T03:39:51,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/643999685d444008aaf9560fc60409f0 is 50, key is test_row_0/C:col10/1731296390331/Put/seqid=0 2024-11-11T03:39:51,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741842_1018 (size=12001) 2024-11-11T03:39:51,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/643999685d444008aaf9560fc60409f0 2024-11-11T03:39:51,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/a28329239071493d82e698a23888d235 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a28329239071493d82e698a23888d235 2024-11-11T03:39:51,451 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a28329239071493d82e698a23888d235, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T03:39:51,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/61bfb881ee374328a668c4fee4d3d72d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/61bfb881ee374328a668c4fee4d3d72d 2024-11-11T03:39:51,477 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/61bfb881ee374328a668c4fee4d3d72d, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T03:39:51,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/643999685d444008aaf9560fc60409f0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/643999685d444008aaf9560fc60409f0 2024-11-11T03:39:51,497 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/643999685d444008aaf9560fc60409f0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T03:39:51,499 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 488bb5013d196b74e2d5128b2d5b7358 in 347ms, sequenceid=33, compaction requested=false 2024-11-11T03:39:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:51,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T03:39:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T03:39:51,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T03:39:51,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3490 sec 2024-11-11T03:39:51,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3800 sec 2024-11-11T03:39:51,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:39:51,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:51,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:51,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:51,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:51,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:51,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:51,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/4b1e9ac7e5b14ced97c150fd4685c99c is 50, key is test_row_0/A:col10/1731296391549/Put/seqid=0 2024-11-11T03:39:51,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741843_1019 (size=11997) 2024-11-11T03:39:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296451718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296451740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296391644, max timestamp=1731296391742Num uniques 859; 2024-11-11T03:39:51,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296391644, max timestamp=1731296391742Num uniques 859; 2024-11-11T03:39:51,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:39:51,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:39:51,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731296391713, max timestamp=1731296391742Num uniques 859; 2024-11-11T03:39:51,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T03:39:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296451853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296451858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296451861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296451862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296451863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296451983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296451983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296451983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:51,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/4b1e9ac7e5b14ced97c150fd4685c99c 2024-11-11T03:39:52,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/8b571cfd332c4407ad74c7046126c90e is 50, key is test_row_0/B:col10/1731296391549/Put/seqid=0 2024-11-11T03:39:52,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296452082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296452094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741844_1020 (size=9657) 2024-11-11T03:39:52,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/8b571cfd332c4407ad74c7046126c90e 2024-11-11T03:39:52,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/09cb51907aae4e89bec03fbb4ba5f26c is 50, key is test_row_0/C:col10/1731296391549/Put/seqid=0 2024-11-11T03:39:52,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741845_1021 (size=9657) 2024-11-11T03:39:52,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/09cb51907aae4e89bec03fbb4ba5f26c 2024-11-11T03:39:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/4b1e9ac7e5b14ced97c150fd4685c99c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4b1e9ac7e5b14ced97c150fd4685c99c 2024-11-11T03:39:52,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4b1e9ac7e5b14ced97c150fd4685c99c, entries=150, sequenceid=43, filesize=11.7 K 2024-11-11T03:39:52,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/8b571cfd332c4407ad74c7046126c90e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8b571cfd332c4407ad74c7046126c90e 2024-11-11T03:39:52,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296452202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296452202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296452202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8b571cfd332c4407ad74c7046126c90e, entries=100, sequenceid=43, filesize=9.4 K 2024-11-11T03:39:52,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/09cb51907aae4e89bec03fbb4ba5f26c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/09cb51907aae4e89bec03fbb4ba5f26c 2024-11-11T03:39:52,242 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T03:39:52,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/09cb51907aae4e89bec03fbb4ba5f26c, entries=100, sequenceid=43, filesize=9.4 K 2024-11-11T03:39:52,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for 488bb5013d196b74e2d5128b2d5b7358 in 688ms, sequenceid=43, compaction requested=true 2024-11-11T03:39:52,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:52,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:39:52,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:52,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:39:52,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:52,266 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:52,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:39:52,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:52,266 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:52,271 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:52,272 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:39:52,273 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,274 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/0910b28ff7dc4c229d00754540d764e8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/61bfb881ee374328a668c4fee4d3d72d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8b571cfd332c4407ad74c7046126c90e] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=32.9 K 2024-11-11T03:39:52,275 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0910b28ff7dc4c229d00754540d764e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731296390200 2024-11-11T03:39:52,276 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:52,278 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 61bfb881ee374328a668c4fee4d3d72d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731296390321 2024-11-11T03:39:52,279 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:39:52,279 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,279 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b571cfd332c4407ad74c7046126c90e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731296391549 2024-11-11T03:39:52,280 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/603c522a70c0454f980b567bae66a7b9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a28329239071493d82e698a23888d235, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4b1e9ac7e5b14ced97c150fd4685c99c] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=37.4 K 2024-11-11T03:39:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T03:39:52,288 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:39:52,289 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:39:52,290 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 603c522a70c0454f980b567bae66a7b9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731296390152 2024-11-11T03:39:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T03:39:52,292 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28329239071493d82e698a23888d235, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731296390321 2024-11-11T03:39:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T03:39:52,294 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:39:52,296 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:39:52,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:39:52,298 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1e9ac7e5b14ced97c150fd4685c99c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731296391539 2024-11-11T03:39:52,347 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#9 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:52,348 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/e0e0f9f75d4a49ccb089caa884b1a1c3 is 50, key is test_row_0/B:col10/1731296391549/Put/seqid=0 2024-11-11T03:39:52,351 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:52,355 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/50e07a830c4b43a8b3cb5a01d1832645 is 50, key is test_row_0/A:col10/1731296391549/Put/seqid=0 2024-11-11T03:39:52,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741846_1022 (size=12104) 2024-11-11T03:39:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T03:39:52,410 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/e0e0f9f75d4a49ccb089caa884b1a1c3 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e0e0f9f75d4a49ccb089caa884b1a1c3 2024-11-11T03:39:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:52,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-11-11T03:39:52,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741847_1023 (size=12104) 2024-11-11T03:39:52,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731296391813, max timestamp=1731296392422Num uniques 1118; 2024-11-11T03:39:52,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-11T03:39:52,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:39:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:52,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:39:52,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:52,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:39:52,440 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into e0e0f9f75d4a49ccb089caa884b1a1c3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:52,441 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:52,444 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296392266; duration=0sec 2024-11-11T03:39:52,445 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:52,445 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:39:52,445 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:52,447 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:52,448 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:39:52,448 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,448 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1369c0b92a443e5aedd91dc7b137549, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/643999685d444008aaf9560fc60409f0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/09cb51907aae4e89bec03fbb4ba5f26c] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=32.9 K 2024-11-11T03:39:52,449 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f1369c0b92a443e5aedd91dc7b137549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731296390200 2024-11-11T03:39:52,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:52,450 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 643999685d444008aaf9560fc60409f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731296390321 2024-11-11T03:39:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,451 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 09cb51907aae4e89bec03fbb4ba5f26c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731296391549 2024-11-11T03:39:52,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/76efcbccc14b463a9a3a911251cb1005 is 50, key is test_row_0/A:col10/1731296392422/Put/seqid=0 2024-11-11T03:39:52,489 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:52,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741848_1024 (size=14341) 2024-11-11T03:39:52,492 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/c3a8aa6191d74a958bf8aaf26862b798 is 50, key is test_row_0/C:col10/1731296391549/Put/seqid=0 2024-11-11T03:39:52,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/76efcbccc14b463a9a3a911251cb1005 2024-11-11T03:39:52,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296452522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296452523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296452527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741849_1025 (size=12104) 2024-11-11T03:39:52,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296452532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2abfc1deefa5495a8b4f8fbb535953ec is 50, key is test_row_0/B:col10/1731296392422/Put/seqid=0 2024-11-11T03:39:52,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296452532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,555 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/c3a8aa6191d74a958bf8aaf26862b798 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c3a8aa6191d74a958bf8aaf26862b798 2024-11-11T03:39:52,569 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into c3a8aa6191d74a958bf8aaf26862b798(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:52,569 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:52,569 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=13, startTime=1731296392266; duration=0sec 2024-11-11T03:39:52,569 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:52,570 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:39:52,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741850_1026 (size=12001) 2024-11-11T03:39:52,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2abfc1deefa5495a8b4f8fbb535953ec 2024-11-11T03:39:52,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/cec5dc9e752a48a784b826dd8d9f6182 is 50, key is test_row_0/C:col10/1731296392422/Put/seqid=0 2024-11-11T03:39:52,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:52,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:52,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T03:39:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741851_1027 (size=12001) 2024-11-11T03:39:52,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296452663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296452652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,844 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/50e07a830c4b43a8b3cb5a01d1832645 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/50e07a830c4b43a8b3cb5a01d1832645 2024-11-11T03:39:52,864 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 50e07a830c4b43a8b3cb5a01d1832645(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:52,864 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:52,864 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296392256; duration=0sec 2024-11-11T03:39:52,865 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:52,865 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:39:52,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296452872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296452883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:52,889 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:39:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T03:39:52,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:52,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:52,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:52,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296453042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296453042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/cec5dc9e752a48a784b826dd8d9f6182 2024-11-11T03:39:53,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296453052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/76efcbccc14b463a9a3a911251cb1005 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/76efcbccc14b463a9a3a911251cb1005 2024-11-11T03:39:53,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/76efcbccc14b463a9a3a911251cb1005, entries=200, sequenceid=82, filesize=14.0 K 2024-11-11T03:39:53,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2abfc1deefa5495a8b4f8fbb535953ec as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2abfc1deefa5495a8b4f8fbb535953ec 2024-11-11T03:39:53,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2abfc1deefa5495a8b4f8fbb535953ec, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T03:39:53,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/cec5dc9e752a48a784b826dd8d9f6182 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cec5dc9e752a48a784b826dd8d9f6182 2024-11-11T03:39:53,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cec5dc9e752a48a784b826dd8d9f6182, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T03:39:53,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~106.95 KB/109515, heapSize ~301.40 KB/308632, currentSize=73.80 KB/75570 for 488bb5013d196b74e2d5128b2d5b7358 in 708ms, sequenceid=82, compaction requested=false 2024-11-11T03:39:53,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:53,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:39:53,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:53,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:53,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:53,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:53,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:53,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:53,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/5e6ca550b6d24e27b13d9a7e338b8762 is 50, key is test_row_0/A:col10/1731296392527/Put/seqid=0 2024-11-11T03:39:53,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741852_1028 (size=14341) 2024-11-11T03:39:53,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/5e6ca550b6d24e27b13d9a7e338b8762 2024-11-11T03:39:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/386d36316a544a468997bdc82cae994b is 50, key is test_row_0/B:col10/1731296392527/Put/seqid=0 2024-11-11T03:39:53,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741853_1029 (size=12001) 2024-11-11T03:39:53,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/386d36316a544a468997bdc82cae994b 2024-11-11T03:39:53,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296453333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296453335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ad69aeb05b204b529c8dbedff7fe5a3f is 50, key is test_row_0/C:col10/1731296392527/Put/seqid=0 2024-11-11T03:39:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741854_1030 (size=12001) 2024-11-11T03:39:53,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:53,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:53,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T03:39:53,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296453443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296453452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296453652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296453665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:53,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ad69aeb05b204b529c8dbedff7fe5a3f 2024-11-11T03:39:53,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/5e6ca550b6d24e27b13d9a7e338b8762 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/5e6ca550b6d24e27b13d9a7e338b8762 2024-11-11T03:39:53,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/5e6ca550b6d24e27b13d9a7e338b8762, entries=200, sequenceid=100, filesize=14.0 K 2024-11-11T03:39:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/386d36316a544a468997bdc82cae994b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/386d36316a544a468997bdc82cae994b 2024-11-11T03:39:53,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/386d36316a544a468997bdc82cae994b, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T03:39:53,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ad69aeb05b204b529c8dbedff7fe5a3f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ad69aeb05b204b529c8dbedff7fe5a3f 2024-11-11T03:39:53,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ad69aeb05b204b529c8dbedff7fe5a3f, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T03:39:53,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:53,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 488bb5013d196b74e2d5128b2d5b7358 in 669ms, sequenceid=100, compaction requested=true 2024-11-11T03:39:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:53,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,866 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:53,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:39:53,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:53,868 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:53,868 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:53,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:39:53,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:53,868 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:39:53,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:39:53,868 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,869 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/50e07a830c4b43a8b3cb5a01d1832645, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/76efcbccc14b463a9a3a911251cb1005, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/5e6ca550b6d24e27b13d9a7e338b8762] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=39.8 K 2024-11-11T03:39:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:53,870 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50e07a830c4b43a8b3cb5a01d1832645, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731296390326 2024-11-11T03:39:53,871 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:53,872 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:39:53,872 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76efcbccc14b463a9a3a911251cb1005, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731296391813 2024-11-11T03:39:53,872 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,872 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e0e0f9f75d4a49ccb089caa884b1a1c3, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2abfc1deefa5495a8b4f8fbb535953ec, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/386d36316a544a468997bdc82cae994b] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=35.3 K 2024-11-11T03:39:53,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:53,876 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e6ca550b6d24e27b13d9a7e338b8762, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731296392489 2024-11-11T03:39:53,877 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e0f9f75d4a49ccb089caa884b1a1c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731296390326 2024-11-11T03:39:53,878 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2abfc1deefa5495a8b4f8fbb535953ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731296391813 2024-11-11T03:39:53,879 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 386d36316a544a468997bdc82cae994b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731296392489 2024-11-11T03:39:53,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T03:39:53,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T03:39:53,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T03:39:53,892 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T03:39:53,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T03:39:53,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T03:39:53,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T03:39:53,895 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T03:39:53,905 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:53,906 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/b3923fc62e9b4eb59b8df31df024648d is 50, key is test_row_0/A:col10/1731296392527/Put/seqid=0 2024-11-11T03:39:53,918 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:53,923 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2a033dc22b314d8cb68a2de4d5c63d81 is 50, key is test_row_0/B:col10/1731296392527/Put/seqid=0 2024-11-11T03:39:53,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741855_1031 (size=12207) 2024-11-11T03:39:53,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741856_1032 (size=12207) 2024-11-11T03:39:53,948 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/b3923fc62e9b4eb59b8df31df024648d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b3923fc62e9b4eb59b8df31df024648d 2024-11-11T03:39:53,964 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into b3923fc62e9b4eb59b8df31df024648d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:53,964 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:53,964 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296393865; duration=0sec 2024-11-11T03:39:53,964 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:53,965 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:39:53,965 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:53,968 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:53,968 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:39:53,968 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:53,969 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2a033dc22b314d8cb68a2de4d5c63d81 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2a033dc22b314d8cb68a2de4d5c63d81 2024-11-11T03:39:53,969 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c3a8aa6191d74a958bf8aaf26862b798, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cec5dc9e752a48a784b826dd8d9f6182, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ad69aeb05b204b529c8dbedff7fe5a3f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=35.3 K 2024-11-11T03:39:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:53,970 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a8aa6191d74a958bf8aaf26862b798, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731296390326 2024-11-11T03:39:53,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:39:53,971 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec5dc9e752a48a784b826dd8d9f6182, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731296391813 2024-11-11T03:39:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:53,971 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad69aeb05b204b529c8dbedff7fe5a3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731296392489 2024-11-11T03:39:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:53,989 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into 2a033dc22b314d8cb68a2de4d5c63d81(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:53,989 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:53,989 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296393867; duration=0sec 2024-11-11T03:39:53,989 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:53,990 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:39:53,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/86f3da9f35fb4beda39c0ff82234d1f6 is 50, key is test_row_0/A:col10/1731296393271/Put/seqid=0 2024-11-11T03:39:54,005 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:54,006 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/62b1a93add964d83a1064a8bf294b922 is 50, key is test_row_0/C:col10/1731296392527/Put/seqid=0 2024-11-11T03:39:54,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:54,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:54,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741857_1033 (size=14341) 2024-11-11T03:39:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741858_1034 (size=12207) 2024-11-11T03:39:54,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296454047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,056 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/62b1a93add964d83a1064a8bf294b922 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/62b1a93add964d83a1064a8bf294b922 2024-11-11T03:39:54,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296454051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296454062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296454062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,071 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into 62b1a93add964d83a1064a8bf294b922(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:54,071 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:54,071 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=13, startTime=1731296393868; duration=0sec 2024-11-11T03:39:54,071 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:54,070 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:39:54,072 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:39:54,070 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:39:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296454072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,076 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:39:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296454162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296454162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296454373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296454373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/86f3da9f35fb4beda39c0ff82234d1f6 2024-11-11T03:39:54,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/5e1260b511ff45f5b3e4398eebd0dc98 is 50, key is test_row_0/B:col10/1731296393271/Put/seqid=0 2024-11-11T03:39:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T03:39:54,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741859_1035 (size=12001) 2024-11-11T03:39:54,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:54,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:54,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296454682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296454682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:54,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/5e1260b511ff45f5b3e4398eebd0dc98 2024-11-11T03:39:54,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/c2e8f8d293004da5b161ecd423846bff is 50, key is test_row_0/C:col10/1731296393271/Put/seqid=0 2024-11-11T03:39:54,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741860_1036 (size=12001) 2024-11-11T03:39:54,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:54,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:55,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:55,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:55,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:55,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296455192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296455202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:55,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:55,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:55,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:55,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/c2e8f8d293004da5b161ecd423846bff 2024-11-11T03:39:55,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/86f3da9f35fb4beda39c0ff82234d1f6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/86f3da9f35fb4beda39c0ff82234d1f6 2024-11-11T03:39:55,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/86f3da9f35fb4beda39c0ff82234d1f6, entries=200, sequenceid=118, filesize=14.0 K 2024-11-11T03:39:55,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/5e1260b511ff45f5b3e4398eebd0dc98 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/5e1260b511ff45f5b3e4398eebd0dc98 2024-11-11T03:39:55,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/5e1260b511ff45f5b3e4398eebd0dc98, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T03:39:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/c2e8f8d293004da5b161ecd423846bff as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c2e8f8d293004da5b161ecd423846bff 2024-11-11T03:39:55,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c2e8f8d293004da5b161ecd423846bff, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T03:39:55,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 488bb5013d196b74e2d5128b2d5b7358 in 1374ms, sequenceid=118, compaction requested=false 2024-11-11T03:39:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:55,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T03:39:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:55,428 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:39:55,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:55,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:55,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:55,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:55,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:55,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8f689ed6102f46b8b20eb8db14c93db6 is 50, key is test_row_0/A:col10/1731296394044/Put/seqid=0 2024-11-11T03:39:55,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741861_1037 (size=12151) 2024-11-11T03:39:55,846 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8f689ed6102f46b8b20eb8db14c93db6 2024-11-11T03:39:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/cace7fcb2b784a6485a7e01b1ac847ce is 50, key is test_row_0/B:col10/1731296394044/Put/seqid=0 2024-11-11T03:39:55,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741862_1038 (size=12151) 2024-11-11T03:39:55,873 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/cace7fcb2b784a6485a7e01b1ac847ce 2024-11-11T03:39:55,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/d2e84a1cf8c4418291d207c2fafd8518 is 50, key is test_row_0/C:col10/1731296394044/Put/seqid=0 2024-11-11T03:39:55,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741863_1039 (size=12151) 2024-11-11T03:39:55,912 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/d2e84a1cf8c4418291d207c2fafd8518 2024-11-11T03:39:55,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8f689ed6102f46b8b20eb8db14c93db6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f689ed6102f46b8b20eb8db14c93db6 2024-11-11T03:39:55,939 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f689ed6102f46b8b20eb8db14c93db6, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T03:39:55,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/cace7fcb2b784a6485a7e01b1ac847ce as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/cace7fcb2b784a6485a7e01b1ac847ce 2024-11-11T03:39:55,955 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/cace7fcb2b784a6485a7e01b1ac847ce, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T03:39:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/d2e84a1cf8c4418291d207c2fafd8518 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d2e84a1cf8c4418291d207c2fafd8518 2024-11-11T03:39:55,967 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d2e84a1cf8c4418291d207c2fafd8518, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T03:39:55,969 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 488bb5013d196b74e2d5128b2d5b7358 in 541ms, sequenceid=135, compaction requested=true 2024-11-11T03:39:55,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:55,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:55,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T03:39:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T03:39:55,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T03:39:55,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6760 sec 2024-11-11T03:39:55,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 3.6900 sec 2024-11-11T03:39:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:56,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:39:56,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:56,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:56,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:56,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:56,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:56,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:56,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/10efa001273b4be2ac705200ba7808c1 is 50, key is test_row_0/A:col10/1731296396106/Put/seqid=0 2024-11-11T03:39:56,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741864_1040 (size=16931) 2024-11-11T03:39:56,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296396135, max timestamp=1731296396168Num uniques 859; 2024-11-11T03:39:56,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:39:56,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296396135, max timestamp=1731296396168Num uniques 859; 2024-11-11T03:39:56,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:39:56,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296396135, max timestamp=1731296396168Num uniques 859; 2024-11-11T03:39:56,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:39:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296456223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296456226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296456228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296456228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296456230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296456343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296456343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296456343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296456343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296456344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T03:39:56,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:39:56,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:39:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T03:39:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T03:39:56,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:39:56,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:39:56,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:39:56,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/10efa001273b4be2ac705200ba7808c1 2024-11-11T03:39:56,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a2c48f06b0e4447b84a5e40bdf65bdd8 is 50, key is test_row_0/B:col10/1731296396106/Put/seqid=0 2024-11-11T03:39:56,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296456552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296456554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296456563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296456564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296456565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T03:39:56,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741865_1041 (size=12151) 2024-11-11T03:39:56,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T03:39:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:56,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T03:39:56,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:56,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:56,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T03:39:56,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296456862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296456872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296456872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296456882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296456882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:56,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T03:39:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:56,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:56,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a2c48f06b0e4447b84a5e40bdf65bdd8 2024-11-11T03:39:56,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/cc494ba7e9eb481c8d3097b73dc441d7 is 50, key is test_row_0/C:col10/1731296396106/Put/seqid=0 2024-11-11T03:39:56,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741866_1042 (size=12151) 2024-11-11T03:39:57,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T03:39:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T03:39:57,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T03:39:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296457372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296457393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296457393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:57,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T03:39:57,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:57,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:57,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:57,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/cc494ba7e9eb481c8d3097b73dc441d7 2024-11-11T03:39:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296457394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:57,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296457394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:57,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/10efa001273b4be2ac705200ba7808c1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/10efa001273b4be2ac705200ba7808c1 2024-11-11T03:39:57,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/10efa001273b4be2ac705200ba7808c1, entries=250, sequenceid=145, filesize=16.5 K 2024-11-11T03:39:57,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a2c48f06b0e4447b84a5e40bdf65bdd8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a2c48f06b0e4447b84a5e40bdf65bdd8 2024-11-11T03:39:57,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a2c48f06b0e4447b84a5e40bdf65bdd8, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T03:39:57,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/cc494ba7e9eb481c8d3097b73dc441d7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cc494ba7e9eb481c8d3097b73dc441d7 2024-11-11T03:39:57,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cc494ba7e9eb481c8d3097b73dc441d7, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T03:39:57,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 488bb5013d196b74e2d5128b2d5b7358 in 1338ms, sequenceid=145, compaction requested=true 2024-11-11T03:39:57,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:57,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:39:57,451 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:39:57,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:57,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:39:57,451 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:39:57,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:57,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:39:57,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:57,453 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:39:57,453 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:39:57,453 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,453 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2a033dc22b314d8cb68a2de4d5c63d81, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/5e1260b511ff45f5b3e4398eebd0dc98, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/cace7fcb2b784a6485a7e01b1ac847ce, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a2c48f06b0e4447b84a5e40bdf65bdd8] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=47.4 K 2024-11-11T03:39:57,454 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55630 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:39:57,454 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a033dc22b314d8cb68a2de4d5c63d81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731296392489 2024-11-11T03:39:57,454 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:39:57,454 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,454 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b3923fc62e9b4eb59b8df31df024648d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/86f3da9f35fb4beda39c0ff82234d1f6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f689ed6102f46b8b20eb8db14c93db6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/10efa001273b4be2ac705200ba7808c1] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=54.3 K 2024-11-11T03:39:57,454 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1260b511ff45f5b3e4398eebd0dc98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731296393271 2024-11-11T03:39:57,455 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3923fc62e9b4eb59b8df31df024648d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731296392489 2024-11-11T03:39:57,455 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting cace7fcb2b784a6485a7e01b1ac847ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296394038 2024-11-11T03:39:57,455 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86f3da9f35fb4beda39c0ff82234d1f6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731296393271 2024-11-11T03:39:57,455 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c48f06b0e4447b84a5e40bdf65bdd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731296396106 2024-11-11T03:39:57,456 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f689ed6102f46b8b20eb8db14c93db6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296394038 2024-11-11T03:39:57,457 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10efa001273b4be2ac705200ba7808c1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731296396099 2024-11-11T03:39:57,482 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:57,483 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/ca8c8f52c31941e7ad4a70e158b20eb1 is 50, key is test_row_0/A:col10/1731296396106/Put/seqid=0 2024-11-11T03:39:57,494 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:57,495 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/b3a8735058fd469a91a3afff52fd425f is 50, key is test_row_0/B:col10/1731296396106/Put/seqid=0 2024-11-11T03:39:57,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741867_1043 (size=12493) 2024-11-11T03:39:57,521 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/ca8c8f52c31941e7ad4a70e158b20eb1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/ca8c8f52c31941e7ad4a70e158b20eb1 2024-11-11T03:39:57,533 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into ca8c8f52c31941e7ad4a70e158b20eb1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:57,533 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:57,533 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=12, startTime=1731296397451; duration=0sec 2024-11-11T03:39:57,533 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:57,533 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:39:57,534 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:39:57,536 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:39:57,536 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:39:57,536 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,536 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/62b1a93add964d83a1064a8bf294b922, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c2e8f8d293004da5b161ecd423846bff, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d2e84a1cf8c4418291d207c2fafd8518, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cc494ba7e9eb481c8d3097b73dc441d7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=47.4 K 2024-11-11T03:39:57,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741868_1044 (size=12493) 2024-11-11T03:39:57,537 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b1a93add964d83a1064a8bf294b922, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731296392489 2024-11-11T03:39:57,538 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e8f8d293004da5b161ecd423846bff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731296393271 2024-11-11T03:39:57,539 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2e84a1cf8c4418291d207c2fafd8518, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296394038 2024-11-11T03:39:57,540 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc494ba7e9eb481c8d3097b73dc441d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731296396106 2024-11-11T03:39:57,547 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/b3a8735058fd469a91a3afff52fd425f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/b3a8735058fd469a91a3afff52fd425f 2024-11-11T03:39:57,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T03:39:57,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:57,550 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-11T03:39:57,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:57,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:39:57,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:57,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:39:57,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:57,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:39:57,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/05735f1b06694868a61bb581048be9b5 is 50, key is test_row_0/A:col10/1731296396227/Put/seqid=0 2024-11-11T03:39:57,565 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into b3a8735058fd469a91a3afff52fd425f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:57,565 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:57,565 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=12, startTime=1731296397451; duration=0sec 2024-11-11T03:39:57,565 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:57,565 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:39:57,574 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:57,575 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f4980bd179ae418b90805b91c40fb0df is 50, key is test_row_0/C:col10/1731296396106/Put/seqid=0 2024-11-11T03:39:57,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741869_1045 (size=12151) 2024-11-11T03:39:57,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741870_1046 (size=12493) 2024-11-11T03:39:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T03:39:57,616 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f4980bd179ae418b90805b91c40fb0df as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f4980bd179ae418b90805b91c40fb0df 2024-11-11T03:39:57,637 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into f4980bd179ae418b90805b91c40fb0df(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:57,637 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:57,637 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=12, startTime=1731296397451; duration=0sec 2024-11-11T03:39:57,638 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:57,638 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:39:57,982 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/05735f1b06694868a61bb581048be9b5 2024-11-11T03:39:58,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/caeeb00c41594fa89c8c45d697c75fd8 is 50, key is test_row_0/B:col10/1731296396227/Put/seqid=0 2024-11-11T03:39:58,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741871_1047 (size=12151) 2024-11-11T03:39:58,052 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/caeeb00c41594fa89c8c45d697c75fd8 2024-11-11T03:39:58,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/eca5b6b317584e2bb58e5f4f9560351b is 50, key is test_row_0/C:col10/1731296396227/Put/seqid=0 2024-11-11T03:39:58,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741872_1048 (size=12151) 2024-11-11T03:39:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:58,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296458412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296458413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296458413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:58,419 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:39:58,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:58,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296458414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296458418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:58,420 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:39:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:58,421 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:39:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:58,483 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/eca5b6b317584e2bb58e5f4f9560351b 2024-11-11T03:39:58,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/05735f1b06694868a61bb581048be9b5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/05735f1b06694868a61bb581048be9b5 2024-11-11T03:39:58,513 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/05735f1b06694868a61bb581048be9b5, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T03:39:58,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/caeeb00c41594fa89c8c45d697c75fd8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/caeeb00c41594fa89c8c45d697c75fd8 2024-11-11T03:39:58,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296458533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:58,537 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/caeeb00c41594fa89c8c45d697c75fd8, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T03:39:58,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296458533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:58,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:58,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/eca5b6b317584e2bb58e5f4f9560351b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eca5b6b317584e2bb58e5f4f9560351b 2024-11-11T03:39:58,552 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eca5b6b317584e2bb58e5f4f9560351b, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T03:39:58,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 488bb5013d196b74e2d5128b2d5b7358 in 1007ms, sequenceid=182, compaction requested=false 2024-11-11T03:39:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T03:39:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T03:39:58,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T03:39:58,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0920 sec 2024-11-11T03:39:58,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.1040 sec 2024-11-11T03:39:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T03:39:58,612 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:39:58,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:39:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T03:39:58,616 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:39:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T03:39:58,618 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:39:58,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:39:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T03:39:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:58,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:39:58,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:58,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:58,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:58,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:58,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:58,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:58,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9bb241c292ea465395ee4da60fccb1b5 is 50, key is test_row_0/A:col10/1731296398746/Put/seqid=0 2024-11-11T03:39:58,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T03:39:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:58,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:58,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741873_1049 (size=12151) 2024-11-11T03:39:58,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9bb241c292ea465395ee4da60fccb1b5 2024-11-11T03:39:58,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/1426810f812242e6bf5a8d2b9b6f3f4a is 50, key is test_row_0/B:col10/1731296398746/Put/seqid=0 2024-11-11T03:39:58,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296398861, max timestamp=1731296398895Num uniques 859; 2024-11-11T03:39:58,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296398861, max timestamp=1731296398895Num uniques 859; 2024-11-11T03:39:58,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:39:58,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:39:58,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296398861, max timestamp=1731296398895Num uniques 859; 2024-11-11T03:39:58,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:39:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T03:39:58,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T03:39:58,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:58,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:58,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:58,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:58,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:58,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741874_1050 (size=12151) 2024-11-11T03:39:58,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/1426810f812242e6bf5a8d2b9b6f3f4a 2024-11-11T03:39:58,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/8f8579ce33ec4caea80ef17fc7944dcd is 50, key is test_row_0/C:col10/1731296398746/Put/seqid=0 2024-11-11T03:39:58,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741875_1051 (size=12151) 2024-11-11T03:39:58,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/8f8579ce33ec4caea80ef17fc7944dcd 2024-11-11T03:39:59,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9bb241c292ea465395ee4da60fccb1b5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9bb241c292ea465395ee4da60fccb1b5 2024-11-11T03:39:59,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296459009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9bb241c292ea465395ee4da60fccb1b5, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T03:39:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296459012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/1426810f812242e6bf5a8d2b9b6f3f4a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/1426810f812242e6bf5a8d2b9b6f3f4a 2024-11-11T03:39:59,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/1426810f812242e6bf5a8d2b9b6f3f4a, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T03:39:59,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/8f8579ce33ec4caea80ef17fc7944dcd as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/8f8579ce33ec4caea80ef17fc7944dcd 2024-11-11T03:39:59,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/8f8579ce33ec4caea80ef17fc7944dcd, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T03:39:59,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 488bb5013d196b74e2d5128b2d5b7358 in 321ms, sequenceid=194, compaction requested=true 2024-11-11T03:39:59,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:59,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:39:59,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:59,071 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:59,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:39:59,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:59,071 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:59,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:39:59,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:59,072 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:59,072 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:59,072 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:39:59,072 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:39:59,072 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:59,072 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:59,073 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/ca8c8f52c31941e7ad4a70e158b20eb1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/05735f1b06694868a61bb581048be9b5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9bb241c292ea465395ee4da60fccb1b5] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=35.9 K 2024-11-11T03:39:59,073 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/b3a8735058fd469a91a3afff52fd425f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/caeeb00c41594fa89c8c45d697c75fd8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/1426810f812242e6bf5a8d2b9b6f3f4a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=35.9 K 2024-11-11T03:39:59,073 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3a8735058fd469a91a3afff52fd425f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731296396106 2024-11-11T03:39:59,073 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8c8f52c31941e7ad4a70e158b20eb1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731296396106 2024-11-11T03:39:59,074 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 05735f1b06694868a61bb581048be9b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731296396206 2024-11-11T03:39:59,074 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting caeeb00c41594fa89c8c45d697c75fd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731296396206 2024-11-11T03:39:59,075 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb241c292ea465395ee4da60fccb1b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296398406 2024-11-11T03:39:59,075 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1426810f812242e6bf5a8d2b9b6f3f4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296398406 2024-11-11T03:39:59,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T03:39:59,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:59,097 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-11T03:39:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:39:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:39:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:39:59,099 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:59,100 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/8acd56e6f6414f50b89c0a17a977fd5f is 50, key is test_row_0/B:col10/1731296398746/Put/seqid=0 2024-11-11T03:39:59,108 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:59,109 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9359bc45d8d9440ba44a571ead594092 is 50, key is test_row_0/A:col10/1731296398746/Put/seqid=0 2024-11-11T03:39:59,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/1ee6bd3de5a5424d8fc7ec8c7239e2de is 50, key is test_row_0/A:col10/1731296399002/Put/seqid=0 2024-11-11T03:39:59,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:59,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741876_1052 (size=12595) 2024-11-11T03:39:59,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741877_1053 (size=12595) 2024-11-11T03:39:59,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741878_1054 (size=12151) 2024-11-11T03:39:59,156 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/1ee6bd3de5a5424d8fc7ec8c7239e2de 2024-11-11T03:39:59,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296459152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296459159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a080a2d2c79a4ca9af6676b84b48408f is 50, key is test_row_0/B:col10/1731296399002/Put/seqid=0 2024-11-11T03:39:59,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741879_1055 (size=12151) 2024-11-11T03:39:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T03:39:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296459262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296459282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296459472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296459491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,539 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/8acd56e6f6414f50b89c0a17a977fd5f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8acd56e6f6414f50b89c0a17a977fd5f 2024-11-11T03:39:59,548 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into 8acd56e6f6414f50b89c0a17a977fd5f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:59,548 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:59,548 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296399071; duration=0sec 2024-11-11T03:39:59,548 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:39:59,548 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:39:59,548 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:39:59,551 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:39:59,551 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:39:59,551 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:59,551 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f4980bd179ae418b90805b91c40fb0df, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eca5b6b317584e2bb58e5f4f9560351b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/8f8579ce33ec4caea80ef17fc7944dcd] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=35.9 K 2024-11-11T03:39:59,552 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4980bd179ae418b90805b91c40fb0df, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731296396106 2024-11-11T03:39:59,556 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca5b6b317584e2bb58e5f4f9560351b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731296396206 2024-11-11T03:39:59,557 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f8579ce33ec4caea80ef17fc7944dcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296398406 2024-11-11T03:39:59,557 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9359bc45d8d9440ba44a571ead594092 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9359bc45d8d9440ba44a571ead594092 2024-11-11T03:39:59,565 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 9359bc45d8d9440ba44a571ead594092(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:39:59,565 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:59,566 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296399070; duration=0sec 2024-11-11T03:39:59,566 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:39:59,566 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:39:59,573 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:39:59,573 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/49770f0689174ea4b421b4715280a561 is 50, key is test_row_0/C:col10/1731296398746/Put/seqid=0 2024-11-11T03:39:59,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741880_1056 (size=12595) 2024-11-11T03:39:59,589 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a080a2d2c79a4ca9af6676b84b48408f 2024-11-11T03:39:59,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/3b4bf88947a142b6b650b802da534115 is 50, key is test_row_0/C:col10/1731296399002/Put/seqid=0 2024-11-11T03:39:59,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741881_1057 (size=12151) 2024-11-11T03:39:59,622 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/3b4bf88947a142b6b650b802da534115 2024-11-11T03:39:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/1ee6bd3de5a5424d8fc7ec8c7239e2de as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/1ee6bd3de5a5424d8fc7ec8c7239e2de 2024-11-11T03:39:59,652 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/1ee6bd3de5a5424d8fc7ec8c7239e2de, entries=150, sequenceid=231, filesize=11.9 K 2024-11-11T03:39:59,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a080a2d2c79a4ca9af6676b84b48408f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a080a2d2c79a4ca9af6676b84b48408f 2024-11-11T03:39:59,676 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a080a2d2c79a4ca9af6676b84b48408f, entries=150, sequenceid=231, filesize=11.9 K 2024-11-11T03:39:59,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/3b4bf88947a142b6b650b802da534115 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/3b4bf88947a142b6b650b802da534115 2024-11-11T03:39:59,696 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/3b4bf88947a142b6b650b802da534115, entries=150, sequenceid=231, filesize=11.9 K 2024-11-11T03:39:59,697 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for 488bb5013d196b74e2d5128b2d5b7358 in 601ms, sequenceid=231, compaction requested=false 2024-11-11T03:39:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:39:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T03:39:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T03:39:59,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T03:39:59,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0810 sec 2024-11-11T03:39:59,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.0890 sec 2024-11-11T03:39:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T03:39:59,753 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:39:59,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:39:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T03:39:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T03:39:59,756 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:39:59,756 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:39:59,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:39:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:39:59,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:39:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:39:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:39:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:39:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:39:59,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/879509d3938949f6ad0c1a1ae4db0a67 is 50, key is test_row_0/A:col10/1731296399784/Put/seqid=0 2024-11-11T03:39:59,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741882_1058 (size=14541) 2024-11-11T03:39:59,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/879509d3938949f6ad0c1a1ae4db0a67 2024-11-11T03:39:59,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/f6508cd46f5d42c380834ba015c5e662 is 50, key is test_row_0/B:col10/1731296399784/Put/seqid=0 2024-11-11T03:39:59,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741883_1059 (size=12151) 2024-11-11T03:39:59,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296399829, max timestamp=1731296399850Num uniques 859; 2024-11-11T03:39:59,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:39:59,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296399829, max timestamp=1731296399850Num uniques 859; 2024-11-11T03:39:59,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296399829, max timestamp=1731296399850Num uniques 859; 2024-11-11T03:39:59,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:39:59,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:39:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T03:39:59,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T03:39:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:39:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:39:59,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:39:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:39:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296459912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296459916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:39:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:39:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:39:59,998 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/49770f0689174ea4b421b4715280a561 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/49770f0689174ea4b421b4715280a561 2024-11-11T03:40:00,013 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into 49770f0689174ea4b421b4715280a561(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:00,013 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:00,013 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=13, startTime=1731296399071; duration=0sec 2024-11-11T03:40:00,013 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:00,013 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:00,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296460032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296460032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T03:40:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T03:40:00,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T03:40:00,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:00,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/f6508cd46f5d42c380834ba015c5e662 2024-11-11T03:40:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296460243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296460243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/e30f74ea2ee14d95a200f8dbf43c78d7 is 50, key is test_row_0/C:col10/1731296399784/Put/seqid=0 2024-11-11T03:40:00,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741884_1060 (size=12151) 2024-11-11T03:40:00,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T03:40:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T03:40:00,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296460432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,440 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296460442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296460443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,446 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,447 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T03:40:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:00,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296460553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296460562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:00,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T03:40:00,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:00,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:00,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/e30f74ea2ee14d95a200f8dbf43c78d7 2024-11-11T03:40:00,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/879509d3938949f6ad0c1a1ae4db0a67 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/879509d3938949f6ad0c1a1ae4db0a67 2024-11-11T03:40:00,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/879509d3938949f6ad0c1a1ae4db0a67, entries=200, sequenceid=243, filesize=14.2 K 2024-11-11T03:40:00,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/f6508cd46f5d42c380834ba015c5e662 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f6508cd46f5d42c380834ba015c5e662 2024-11-11T03:40:00,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f6508cd46f5d42c380834ba015c5e662, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T03:40:00,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/e30f74ea2ee14d95a200f8dbf43c78d7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e30f74ea2ee14d95a200f8dbf43c78d7 2024-11-11T03:40:00,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e30f74ea2ee14d95a200f8dbf43c78d7, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T03:40:00,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 488bb5013d196b74e2d5128b2d5b7358 in 955ms, sequenceid=243, compaction requested=true 2024-11-11T03:40:00,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:00,741 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:00,743 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:00,743 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:40:00,743 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,743 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9359bc45d8d9440ba44a571ead594092, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/1ee6bd3de5a5424d8fc7ec8c7239e2de, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/879509d3938949f6ad0c1a1ae4db0a67] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=38.4 K 2024-11-11T03:40:00,744 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9359bc45d8d9440ba44a571ead594092, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296398406 2024-11-11T03:40:00,744 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee6bd3de5a5424d8fc7ec8c7239e2de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731296398984 2024-11-11T03:40:00,745 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879509d3938949f6ad0c1a1ae4db0a67, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731296399147 2024-11-11T03:40:00,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:00,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:00,747 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:00,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:00,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:00,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:00,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:00,749 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:00,749 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:40:00,749 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,749 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8acd56e6f6414f50b89c0a17a977fd5f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a080a2d2c79a4ca9af6676b84b48408f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f6508cd46f5d42c380834ba015c5e662] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=36.0 K 2024-11-11T03:40:00,749 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8acd56e6f6414f50b89c0a17a977fd5f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296398406 2024-11-11T03:40:00,750 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting a080a2d2c79a4ca9af6676b84b48408f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731296398984 2024-11-11T03:40:00,751 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f6508cd46f5d42c380834ba015c5e662, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731296399147 2024-11-11T03:40:00,763 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:00,763 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0f1cd61eb5e54b0fbbf42abeeb14b83f is 50, key is test_row_0/A:col10/1731296399784/Put/seqid=0 2024-11-11T03:40:00,768 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:00,769 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/bbdf53628eae41d9a6c71de8fd70954a is 50, key is test_row_0/B:col10/1731296399784/Put/seqid=0 2024-11-11T03:40:00,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741885_1061 (size=12697) 2024-11-11T03:40:00,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741886_1062 (size=12697) 2024-11-11T03:40:00,812 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0f1cd61eb5e54b0fbbf42abeeb14b83f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0f1cd61eb5e54b0fbbf42abeeb14b83f 2024-11-11T03:40:00,822 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 0f1cd61eb5e54b0fbbf42abeeb14b83f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:00,822 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:00,822 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296400741; duration=0sec 2024-11-11T03:40:00,822 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:00,822 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:00,822 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:00,824 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:00,824 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:40:00,824 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,825 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/49770f0689174ea4b421b4715280a561, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/3b4bf88947a142b6b650b802da534115, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e30f74ea2ee14d95a200f8dbf43c78d7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=36.0 K 2024-11-11T03:40:00,825 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49770f0689174ea4b421b4715280a561, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296398406 2024-11-11T03:40:00,826 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b4bf88947a142b6b650b802da534115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731296398984 2024-11-11T03:40:00,826 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e30f74ea2ee14d95a200f8dbf43c78d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731296399147 2024-11-11T03:40:00,835 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:00,836 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/2bb417c5ee764f8e91711851c3d341da is 50, key is test_row_0/C:col10/1731296399784/Put/seqid=0 2024-11-11T03:40:00,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741887_1063 (size=12697) 2024-11-11T03:40:00,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T03:40:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:00,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T03:40:00,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:00,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:00,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:00,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:00,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:00,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:00,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0e2b3a6374154f0da882087258701af4 is 50, key is test_row_0/A:col10/1731296399899/Put/seqid=0 2024-11-11T03:40:00,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741888_1064 (size=12301) 2024-11-11T03:40:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T03:40:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:01,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296461085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296461091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,196 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/bbdf53628eae41d9a6c71de8fd70954a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/bbdf53628eae41d9a6c71de8fd70954a 2024-11-11T03:40:01,203 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into bbdf53628eae41d9a6c71de8fd70954a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:01,203 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:01,203 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296400746; duration=0sec 2024-11-11T03:40:01,204 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:01,204 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:01,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296461202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296461202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,250 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/2bb417c5ee764f8e91711851c3d341da as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/2bb417c5ee764f8e91711851c3d341da 2024-11-11T03:40:01,258 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into 2bb417c5ee764f8e91711851c3d341da(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:01,259 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:01,259 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=13, startTime=1731296400747; duration=0sec 2024-11-11T03:40:01,259 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:01,259 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:01,266 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0e2b3a6374154f0da882087258701af4 2024-11-11T03:40:01,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/51f626f1635f44949d115db3fcca6cd8 is 50, key is test_row_0/B:col10/1731296399899/Put/seqid=0 2024-11-11T03:40:01,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741889_1065 (size=12301) 2024-11-11T03:40:01,293 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/51f626f1635f44949d115db3fcca6cd8 2024-11-11T03:40:01,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/bc3efe93a5634099b69bf903b818ab3d is 50, key is test_row_0/C:col10/1731296399899/Put/seqid=0 2024-11-11T03:40:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741890_1066 (size=12301) 2024-11-11T03:40:01,335 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/bc3efe93a5634099b69bf903b818ab3d 2024-11-11T03:40:01,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0e2b3a6374154f0da882087258701af4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0e2b3a6374154f0da882087258701af4 2024-11-11T03:40:01,365 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0e2b3a6374154f0da882087258701af4, entries=150, sequenceid=281, filesize=12.0 K 2024-11-11T03:40:01,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/51f626f1635f44949d115db3fcca6cd8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/51f626f1635f44949d115db3fcca6cd8 2024-11-11T03:40:01,384 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/51f626f1635f44949d115db3fcca6cd8, entries=150, sequenceid=281, filesize=12.0 K 2024-11-11T03:40:01,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/bc3efe93a5634099b69bf903b818ab3d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bc3efe93a5634099b69bf903b818ab3d 2024-11-11T03:40:01,405 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bc3efe93a5634099b69bf903b818ab3d, entries=150, sequenceid=281, filesize=12.0 K 2024-11-11T03:40:01,408 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 488bb5013d196b74e2d5128b2d5b7358 in 561ms, sequenceid=281, compaction requested=false 2024-11-11T03:40:01,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:01,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:01,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T03:40:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T03:40:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:01,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T03:40:01,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T03:40:01,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6530 sec 2024-11-11T03:40:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:01,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6710 sec 2024-11-11T03:40:01,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/e059286810ec42e892e05b48dc845e12 is 50, key is test_row_0/A:col10/1731296401415/Put/seqid=0 2024-11-11T03:40:01,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741891_1067 (size=17181) 2024-11-11T03:40:01,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/e059286810ec42e892e05b48dc845e12 2024-11-11T03:40:01,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/07355ab769f44ec78cce2b7757485ed4 is 50, key is test_row_0/B:col10/1731296401415/Put/seqid=0 2024-11-11T03:40:01,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741892_1068 (size=12301) 2024-11-11T03:40:01,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/07355ab769f44ec78cce2b7757485ed4 2024-11-11T03:40:01,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296461515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296461518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ecf50f58d18b4b7c86de9ec65f79ef62 is 50, key is test_row_0/C:col10/1731296401415/Put/seqid=0 2024-11-11T03:40:01,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741893_1069 (size=12301) 2024-11-11T03:40:01,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ecf50f58d18b4b7c86de9ec65f79ef62 2024-11-11T03:40:01,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/e059286810ec42e892e05b48dc845e12 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/e059286810ec42e892e05b48dc845e12 2024-11-11T03:40:01,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/e059286810ec42e892e05b48dc845e12, entries=250, sequenceid=295, filesize=16.8 K 2024-11-11T03:40:01,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/07355ab769f44ec78cce2b7757485ed4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/07355ab769f44ec78cce2b7757485ed4 2024-11-11T03:40:01,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/07355ab769f44ec78cce2b7757485ed4, entries=150, sequenceid=295, filesize=12.0 K 2024-11-11T03:40:01,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ecf50f58d18b4b7c86de9ec65f79ef62 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ecf50f58d18b4b7c86de9ec65f79ef62 2024-11-11T03:40:01,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ecf50f58d18b4b7c86de9ec65f79ef62, entries=150, sequenceid=295, filesize=12.0 K 2024-11-11T03:40:01,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 488bb5013d196b74e2d5128b2d5b7358 in 176ms, sequenceid=295, compaction requested=true 2024-11-11T03:40:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:01,592 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:01,592 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:01,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:01,593 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42179 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:01,593 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:01,593 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:40:01,593 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:40:01,594 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:01,594 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:01,594 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0f1cd61eb5e54b0fbbf42abeeb14b83f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0e2b3a6374154f0da882087258701af4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/e059286810ec42e892e05b48dc845e12] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=41.2 K 2024-11-11T03:40:01,594 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/bbdf53628eae41d9a6c71de8fd70954a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/51f626f1635f44949d115db3fcca6cd8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/07355ab769f44ec78cce2b7757485ed4] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=36.4 K 2024-11-11T03:40:01,594 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1cd61eb5e54b0fbbf42abeeb14b83f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731296399147 2024-11-11T03:40:01,594 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting bbdf53628eae41d9a6c71de8fd70954a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731296399147 2024-11-11T03:40:01,594 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 51f626f1635f44949d115db3fcca6cd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731296399899 2024-11-11T03:40:01,594 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e2b3a6374154f0da882087258701af4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731296399899 2024-11-11T03:40:01,595 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 07355ab769f44ec78cce2b7757485ed4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731296401079 2024-11-11T03:40:01,595 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e059286810ec42e892e05b48dc845e12, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731296401073 2024-11-11T03:40:01,608 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:01,609 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/2a17d1bf16d546e898b7e47e005a8d6b is 50, key is test_row_0/A:col10/1731296401415/Put/seqid=0 2024-11-11T03:40:01,617 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:01,617 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/41ff48bef1894c8ca696d77d25b52582 is 50, key is test_row_0/B:col10/1731296401415/Put/seqid=0 2024-11-11T03:40:01,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T03:40:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:01,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741894_1070 (size=12949) 2024-11-11T03:40:01,648 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/2a17d1bf16d546e898b7e47e005a8d6b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/2a17d1bf16d546e898b7e47e005a8d6b 2024-11-11T03:40:01,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741895_1071 (size=12949) 2024-11-11T03:40:01,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/58beb69e9a3a40a1b5bdadd0d35d27c8 is 50, key is test_row_0/A:col10/1731296401512/Put/seqid=0 2024-11-11T03:40:01,661 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 2a17d1bf16d546e898b7e47e005a8d6b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:01,661 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:01,661 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296401592; duration=0sec 2024-11-11T03:40:01,662 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:01,662 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:01,662 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:01,663 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:01,663 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:40:01,663 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:01,663 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/2bb417c5ee764f8e91711851c3d341da, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bc3efe93a5634099b69bf903b818ab3d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ecf50f58d18b4b7c86de9ec65f79ef62] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=36.4 K 2024-11-11T03:40:01,664 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bb417c5ee764f8e91711851c3d341da, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731296399147 2024-11-11T03:40:01,667 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc3efe93a5634099b69bf903b818ab3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731296399899 2024-11-11T03:40:01,670 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf50f58d18b4b7c86de9ec65f79ef62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731296401079 2024-11-11T03:40:01,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296461670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296461670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741896_1072 (size=12301) 2024-11-11T03:40:01,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/58beb69e9a3a40a1b5bdadd0d35d27c8 2024-11-11T03:40:01,693 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:01,693 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/d0e85fc02d414f4086f94d0e9775e02a is 50, key is test_row_0/C:col10/1731296401415/Put/seqid=0 2024-11-11T03:40:01,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/10c8466b898343cb9de2896611c5b5e2 is 50, key is test_row_0/B:col10/1731296401512/Put/seqid=0 2024-11-11T03:40:01,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741897_1073 (size=12949) 2024-11-11T03:40:01,721 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/d0e85fc02d414f4086f94d0e9775e02a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d0e85fc02d414f4086f94d0e9775e02a 2024-11-11T03:40:01,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741898_1074 (size=12301) 2024-11-11T03:40:01,731 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into d0e85fc02d414f4086f94d0e9775e02a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:01,731 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:01,731 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=13, startTime=1731296401592; duration=0sec 2024-11-11T03:40:01,731 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:01,731 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296461782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296461792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T03:40:01,902 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:01,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T03:40:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T03:40:01,915 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:01,919 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:01,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:01,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296461992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:02,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296462002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T03:40:02,055 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/41ff48bef1894c8ca696d77d25b52582 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/41ff48bef1894c8ca696d77d25b52582 2024-11-11T03:40:02,061 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into 41ff48bef1894c8ca696d77d25b52582(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:02,061 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:02,062 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296401592; duration=0sec 2024-11-11T03:40:02,062 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:02,062 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:02,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T03:40:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/10c8466b898343cb9de2896611c5b5e2 2024-11-11T03:40:02,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/b07582b1ac7d47fdbe7e244832af9611 is 50, key is test_row_0/C:col10/1731296401512/Put/seqid=0 2024-11-11T03:40:02,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741899_1075 (size=12301) 2024-11-11T03:40:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T03:40:02,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T03:40:02,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:02,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296462302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:02,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296462312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:02,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T03:40:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T03:40:02,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T03:40:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:02,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:02,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/b07582b1ac7d47fdbe7e244832af9611 2024-11-11T03:40:02,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/58beb69e9a3a40a1b5bdadd0d35d27c8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/58beb69e9a3a40a1b5bdadd0d35d27c8 2024-11-11T03:40:02,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/58beb69e9a3a40a1b5bdadd0d35d27c8, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T03:40:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/10c8466b898343cb9de2896611c5b5e2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/10c8466b898343cb9de2896611c5b5e2 2024-11-11T03:40:02,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/10c8466b898343cb9de2896611c5b5e2, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T03:40:02,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/b07582b1ac7d47fdbe7e244832af9611 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b07582b1ac7d47fdbe7e244832af9611 2024-11-11T03:40:02,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b07582b1ac7d47fdbe7e244832af9611, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T03:40:02,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 488bb5013d196b74e2d5128b2d5b7358 in 962ms, sequenceid=316, compaction requested=false 2024-11-11T03:40:02,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:02,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T03:40:02,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:02,688 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T03:40:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:02,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/6245b0b1139846758f84e5551f1ce41e is 50, key is test_row_0/A:col10/1731296401667/Put/seqid=0 2024-11-11T03:40:02,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741900_1076 (size=12301) 2024-11-11T03:40:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:02,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:02,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296402861, max timestamp=1731296402892Num uniques 859; 2024-11-11T03:40:02,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:02,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296402861, max timestamp=1731296402892Num uniques 859; 2024-11-11T03:40:02,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296402861, max timestamp=1731296402892Num uniques 859; 2024-11-11T03:40:02,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:02,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:02,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296462968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296462972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:02,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T03:40:03,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296463081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:03,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:03,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296463081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:03,125 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/6245b0b1139846758f84e5551f1ce41e 2024-11-11T03:40:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/388b2a4fdcac48f3bdb90648b5f5f1c0 is 50, key is test_row_0/B:col10/1731296401667/Put/seqid=0 2024-11-11T03:40:03,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741901_1077 (size=12301) 2024-11-11T03:40:03,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296463292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:03,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296463292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:03,563 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/388b2a4fdcac48f3bdb90648b5f5f1c0 2024-11-11T03:40:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/43c2f6a9e4d84570b9b2981c75c0f094 is 50, key is test_row_0/C:col10/1731296401667/Put/seqid=0 2024-11-11T03:40:03,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741902_1078 (size=12301) 2024-11-11T03:40:03,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:03,604 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/43c2f6a9e4d84570b9b2981c75c0f094 2024-11-11T03:40:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296463602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296463603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:03,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/6245b0b1139846758f84e5551f1ce41e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/6245b0b1139846758f84e5551f1ce41e 2024-11-11T03:40:03,630 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/6245b0b1139846758f84e5551f1ce41e, entries=150, sequenceid=330, filesize=12.0 K 2024-11-11T03:40:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/388b2a4fdcac48f3bdb90648b5f5f1c0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/388b2a4fdcac48f3bdb90648b5f5f1c0 2024-11-11T03:40:03,641 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/388b2a4fdcac48f3bdb90648b5f5f1c0, entries=150, sequenceid=330, filesize=12.0 K 2024-11-11T03:40:03,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/43c2f6a9e4d84570b9b2981c75c0f094 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/43c2f6a9e4d84570b9b2981c75c0f094 2024-11-11T03:40:03,648 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/43c2f6a9e4d84570b9b2981c75c0f094, entries=150, sequenceid=330, filesize=12.0 K 2024-11-11T03:40:03,650 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 488bb5013d196b74e2d5128b2d5b7358 in 961ms, sequenceid=330, compaction requested=true 2024-11-11T03:40:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T03:40:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T03:40:03,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T03:40:03,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7320 sec 2024-11-11T03:40:03,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7520 sec 2024-11-11T03:40:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T03:40:04,053 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:04,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T03:40:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T03:40:04,055 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:04,056 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:04,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:04,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T03:40:04,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:04,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:04,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:04,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:04,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:04,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:04,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/128872c0b26942fabae2e09302435c43 is 50, key is test_row_0/A:col10/1731296402965/Put/seqid=0 2024-11-11T03:40:04,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296464147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296464150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T03:40:04,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741903_1079 (size=14741) 2024-11-11T03:40:04,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/128872c0b26942fabae2e09302435c43 2024-11-11T03:40:04,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/9a735f880da8427a916073c11ffb24aa is 50, key is test_row_0/B:col10/1731296402965/Put/seqid=0 2024-11-11T03:40:04,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741904_1080 (size=12301) 2024-11-11T03:40:04,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/9a735f880da8427a916073c11ffb24aa 2024-11-11T03:40:04,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/e93e47b080af45778eb73b6c3fe54af5 is 50, key is test_row_0/C:col10/1731296402965/Put/seqid=0 2024-11-11T03:40:04,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741905_1081 (size=12301) 2024-11-11T03:40:04,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296464262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296464262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T03:40:04,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48160 deadline: 1731296464452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,453 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731296464472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296464472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,475 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:04,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296464472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48200 deadline: 1731296464482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,485 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 488bb5013d196b74e2d5128b2d5b7358 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/e93e47b080af45778eb73b6c3fe54af5 2024-11-11T03:40:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/128872c0b26942fabae2e09302435c43 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/128872c0b26942fabae2e09302435c43 2024-11-11T03:40:04,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:04,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:04,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/128872c0b26942fabae2e09302435c43, entries=200, sequenceid=366, filesize=14.4 K 2024-11-11T03:40:04,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/9a735f880da8427a916073c11ffb24aa as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/9a735f880da8427a916073c11ffb24aa 2024-11-11T03:40:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T03:40:04,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/9a735f880da8427a916073c11ffb24aa, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T03:40:04,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/e93e47b080af45778eb73b6c3fe54af5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e93e47b080af45778eb73b6c3fe54af5 2024-11-11T03:40:04,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e93e47b080af45778eb73b6c3fe54af5, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T03:40:04,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 488bb5013d196b74e2d5128b2d5b7358 in 575ms, sequenceid=366, compaction requested=true 2024-11-11T03:40:04,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:04,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:04,694 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:04,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:04,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:04,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:04,694 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:04,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:04,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:04,695 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:04,696 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:40:04,696 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,696 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/41ff48bef1894c8ca696d77d25b52582, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/10c8466b898343cb9de2896611c5b5e2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/388b2a4fdcac48f3bdb90648b5f5f1c0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/9a735f880da8427a916073c11ffb24aa] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=48.7 K 2024-11-11T03:40:04,696 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:04,696 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:40:04,696 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 41ff48bef1894c8ca696d77d25b52582, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731296401079 2024-11-11T03:40:04,696 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,697 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/2a17d1bf16d546e898b7e47e005a8d6b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/58beb69e9a3a40a1b5bdadd0d35d27c8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/6245b0b1139846758f84e5551f1ce41e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/128872c0b26942fabae2e09302435c43] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=51.1 K 2024-11-11T03:40:04,697 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c8466b898343cb9de2896611c5b5e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731296401510 2024-11-11T03:40:04,697 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a17d1bf16d546e898b7e47e005a8d6b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731296401079 2024-11-11T03:40:04,697 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 388b2a4fdcac48f3bdb90648b5f5f1c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731296401652 2024-11-11T03:40:04,697 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58beb69e9a3a40a1b5bdadd0d35d27c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731296401510 2024-11-11T03:40:04,697 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a735f880da8427a916073c11ffb24aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731296402957 2024-11-11T03:40:04,698 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6245b0b1139846758f84e5551f1ce41e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731296401652 2024-11-11T03:40:04,698 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128872c0b26942fabae2e09302435c43, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731296402957 2024-11-11T03:40:04,712 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:04,713 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/407c2bc62f8e4f9f880eed7e66e5f450 is 50, key is test_row_0/B:col10/1731296402965/Put/seqid=0 2024-11-11T03:40:04,722 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#70 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:04,723 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/4e28477cfe1d4b11b72ef272bd139dad is 50, key is test_row_0/A:col10/1731296402965/Put/seqid=0 2024-11-11T03:40:04,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741906_1082 (size=13085) 2024-11-11T03:40:04,739 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/407c2bc62f8e4f9f880eed7e66e5f450 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/407c2bc62f8e4f9f880eed7e66e5f450 2024-11-11T03:40:04,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741907_1083 (size=13085) 2024-11-11T03:40:04,754 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into 407c2bc62f8e4f9f880eed7e66e5f450(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:04,754 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:04,754 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=12, startTime=1731296404694; duration=0sec 2024-11-11T03:40:04,754 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:04,754 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:04,754 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:04,758 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/4e28477cfe1d4b11b72ef272bd139dad as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4e28477cfe1d4b11b72ef272bd139dad 2024-11-11T03:40:04,758 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:04,759 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:40:04,759 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,759 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d0e85fc02d414f4086f94d0e9775e02a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b07582b1ac7d47fdbe7e244832af9611, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/43c2f6a9e4d84570b9b2981c75c0f094, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e93e47b080af45778eb73b6c3fe54af5] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=48.7 K 2024-11-11T03:40:04,760 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e85fc02d414f4086f94d0e9775e02a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731296401079 2024-11-11T03:40:04,763 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b07582b1ac7d47fdbe7e244832af9611, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731296401510 2024-11-11T03:40:04,765 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c2f6a9e4d84570b9b2981c75c0f094, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731296401652 2024-11-11T03:40:04,765 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e93e47b080af45778eb73b6c3fe54af5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731296402957 2024-11-11T03:40:04,768 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 4e28477cfe1d4b11b72ef272bd139dad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:04,768 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:04,768 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=12, startTime=1731296404694; duration=0sec 2024-11-11T03:40:04,768 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:04,768 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:04,781 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:04,781 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/78c052439537498b881517d1f58de416 is 50, key is test_row_0/C:col10/1731296402965/Put/seqid=0 2024-11-11T03:40:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:04,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T03:40:04,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:04,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:04,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:04,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:04,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:04,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:04,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/870ee0d5553d4822b268fcbe04a59665 is 50, key is test_row_0/A:col10/1731296404785/Put/seqid=0 2024-11-11T03:40:04,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741908_1084 (size=13085) 2024-11-11T03:40:04,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:04,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741909_1085 (size=14741) 2024-11-11T03:40:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/870ee0d5553d4822b268fcbe04a59665 2024-11-11T03:40:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/f64846bf826f4daeb5d13335b3569a8b is 50, key is test_row_0/B:col10/1731296404785/Put/seqid=0 2024-11-11T03:40:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296464838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741910_1086 (size=12301) 2024-11-11T03:40:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/f64846bf826f4daeb5d13335b3569a8b 2024-11-11T03:40:04,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296464839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/89b3916ade6d4dc3988b54aeae27c718 is 50, key is test_row_0/C:col10/1731296404785/Put/seqid=0 2024-11-11T03:40:04,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741911_1087 (size=12301) 2024-11-11T03:40:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296464952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296464962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:04,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:04,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:05,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:05,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296465162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296465172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T03:40:05,214 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/78c052439537498b881517d1f58de416 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/78c052439537498b881517d1f58de416 2024-11-11T03:40:05,221 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into 78c052439537498b881517d1f58de416(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:05,221 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:05,221 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=12, startTime=1731296404694; duration=0sec 2024-11-11T03:40:05,221 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:05,221 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:05,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/89b3916ade6d4dc3988b54aeae27c718 2024-11-11T03:40:05,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/870ee0d5553d4822b268fcbe04a59665 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/870ee0d5553d4822b268fcbe04a59665 2024-11-11T03:40:05,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:05,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:05,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/870ee0d5553d4822b268fcbe04a59665, entries=200, sequenceid=380, filesize=14.4 K 2024-11-11T03:40:05,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/f64846bf826f4daeb5d13335b3569a8b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f64846bf826f4daeb5d13335b3569a8b 2024-11-11T03:40:05,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f64846bf826f4daeb5d13335b3569a8b, entries=150, sequenceid=380, filesize=12.0 K 2024-11-11T03:40:05,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/89b3916ade6d4dc3988b54aeae27c718 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/89b3916ade6d4dc3988b54aeae27c718 2024-11-11T03:40:05,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/89b3916ade6d4dc3988b54aeae27c718, entries=150, sequenceid=380, filesize=12.0 K 2024-11-11T03:40:05,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 488bb5013d196b74e2d5128b2d5b7358 in 524ms, sequenceid=380, compaction requested=false 2024-11-11T03:40:05,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:05,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T03:40:05,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:05,444 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T03:40:05,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:05,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/7385477af4324ac9abbd8d82a4b6c578 is 50, key is test_row_0/A:col10/1731296404836/Put/seqid=0 2024-11-11T03:40:05,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741912_1088 (size=12301) 2024-11-11T03:40:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296465492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296465493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296465602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296465602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296465812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296465812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:05,857 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/7385477af4324ac9abbd8d82a4b6c578 2024-11-11T03:40:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/615d642e511a42ea83e403182ae905c6 is 50, key is test_row_0/B:col10/1731296404836/Put/seqid=0 2024-11-11T03:40:05,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741913_1089 (size=12301) 2024-11-11T03:40:05,904 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/615d642e511a42ea83e403182ae905c6 2024-11-11T03:40:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/686ad96e6c5949fdb0abbcd4ec4f2be0 is 50, key is test_row_0/C:col10/1731296404836/Put/seqid=0 2024-11-11T03:40:05,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741914_1090 (size=12301) 2024-11-11T03:40:05,941 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/686ad96e6c5949fdb0abbcd4ec4f2be0 2024-11-11T03:40:05,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/7385477af4324ac9abbd8d82a4b6c578 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/7385477af4324ac9abbd8d82a4b6c578 2024-11-11T03:40:05,956 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/7385477af4324ac9abbd8d82a4b6c578, entries=150, sequenceid=401, filesize=12.0 K 2024-11-11T03:40:05,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/615d642e511a42ea83e403182ae905c6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/615d642e511a42ea83e403182ae905c6 2024-11-11T03:40:05,967 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/615d642e511a42ea83e403182ae905c6, entries=150, sequenceid=401, filesize=12.0 K 2024-11-11T03:40:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/686ad96e6c5949fdb0abbcd4ec4f2be0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/686ad96e6c5949fdb0abbcd4ec4f2be0 2024-11-11T03:40:05,978 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/686ad96e6c5949fdb0abbcd4ec4f2be0, entries=150, sequenceid=401, filesize=12.0 K 2024-11-11T03:40:05,980 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 488bb5013d196b74e2d5128b2d5b7358 in 535ms, sequenceid=401, compaction requested=true 2024-11-11T03:40:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T03:40:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T03:40:05,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T03:40:05,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9270 sec 2024-11-11T03:40:05,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9340 sec 2024-11-11T03:40:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:06,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T03:40:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:06,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9eecae8310bd4f06aa0816ab771307e6 is 50, key is test_row_0/A:col10/1731296406123/Put/seqid=0 2024-11-11T03:40:06,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741915_1091 (size=12301) 2024-11-11T03:40:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296466172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296466176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T03:40:06,202 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:06,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T03:40:06,203 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T03:40:06,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:06,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:06,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296466282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296466282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T03:40:06,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T03:40:06,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:06,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296466492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296466492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T03:40:06,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:06,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T03:40:06,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9eecae8310bd4f06aa0816ab771307e6 2024-11-11T03:40:06,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/ce1a23298b114a25b12eeefcbd2b9c92 is 50, key is test_row_0/B:col10/1731296406123/Put/seqid=0 2024-11-11T03:40:06,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741916_1092 (size=12301) 2024-11-11T03:40:06,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T03:40:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296466802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296466802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:06,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T03:40:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T03:40:06,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/ce1a23298b114a25b12eeefcbd2b9c92 2024-11-11T03:40:06,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T03:40:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:06,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:06,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ee387ef6fa8244289f152fa5d90b612f is 50, key is test_row_0/C:col10/1731296406123/Put/seqid=0 2024-11-11T03:40:06,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741917_1093 (size=12301) 2024-11-11T03:40:06,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ee387ef6fa8244289f152fa5d90b612f 2024-11-11T03:40:06,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/9eecae8310bd4f06aa0816ab771307e6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9eecae8310bd4f06aa0816ab771307e6 2024-11-11T03:40:07,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9eecae8310bd4f06aa0816ab771307e6, entries=150, sequenceid=414, filesize=12.0 K 2024-11-11T03:40:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/ce1a23298b114a25b12eeefcbd2b9c92 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ce1a23298b114a25b12eeefcbd2b9c92 2024-11-11T03:40:07,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ce1a23298b114a25b12eeefcbd2b9c92, entries=150, sequenceid=414, filesize=12.0 K 2024-11-11T03:40:07,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ee387ef6fa8244289f152fa5d90b612f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ee387ef6fa8244289f152fa5d90b612f 2024-11-11T03:40:07,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ee387ef6fa8244289f152fa5d90b612f, entries=150, sequenceid=414, filesize=12.0 K 2024-11-11T03:40:07,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 488bb5013d196b74e2d5128b2d5b7358 in 902ms, sequenceid=414, compaction requested=true 2024-11-11T03:40:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:07,026 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:07,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:07,027 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:07,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:07,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:07,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:07,028 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52428 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:07,028 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:40:07,029 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:07,029 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4e28477cfe1d4b11b72ef272bd139dad, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/870ee0d5553d4822b268fcbe04a59665, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/7385477af4324ac9abbd8d82a4b6c578, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9eecae8310bd4f06aa0816ab771307e6] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=51.2 K 2024-11-11T03:40:07,030 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:07,030 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:40:07,030 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:07,030 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/407c2bc62f8e4f9f880eed7e66e5f450, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f64846bf826f4daeb5d13335b3569a8b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/615d642e511a42ea83e403182ae905c6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ce1a23298b114a25b12eeefcbd2b9c92] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=48.8 K 2024-11-11T03:40:07,030 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e28477cfe1d4b11b72ef272bd139dad, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731296402957 2024-11-11T03:40:07,031 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 407c2bc62f8e4f9f880eed7e66e5f450, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731296402957 2024-11-11T03:40:07,031 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 870ee0d5553d4822b268fcbe04a59665, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731296404144 2024-11-11T03:40:07,032 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f64846bf826f4daeb5d13335b3569a8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731296404146 2024-11-11T03:40:07,032 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7385477af4324ac9abbd8d82a4b6c578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731296404831 2024-11-11T03:40:07,032 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 615d642e511a42ea83e403182ae905c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731296404831 2024-11-11T03:40:07,032 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eecae8310bd4f06aa0816ab771307e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731296405490 2024-11-11T03:40:07,033 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1a23298b114a25b12eeefcbd2b9c92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731296405490 2024-11-11T03:40:07,048 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:07,048 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8f0f7639a0ea42f9b45745661d8b5820 is 50, key is test_row_0/A:col10/1731296406123/Put/seqid=0 2024-11-11T03:40:07,061 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:07,062 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/3c54907b8f0b4bcfa1197dcadb5f60b8 is 50, key is test_row_0/B:col10/1731296406123/Put/seqid=0 2024-11-11T03:40:07,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741918_1094 (size=13221) 2024-11-11T03:40:07,101 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8f0f7639a0ea42f9b45745661d8b5820 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f0f7639a0ea42f9b45745661d8b5820 2024-11-11T03:40:07,110 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 8f0f7639a0ea42f9b45745661d8b5820(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:07,110 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:07,110 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=12, startTime=1731296407026; duration=0sec 2024-11-11T03:40:07,110 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:07,110 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:07,110 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741919_1095 (size=13221) 2024-11-11T03:40:07,115 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:07,115 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:40:07,115 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:07,117 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/78c052439537498b881517d1f58de416, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/89b3916ade6d4dc3988b54aeae27c718, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/686ad96e6c5949fdb0abbcd4ec4f2be0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ee387ef6fa8244289f152fa5d90b612f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=48.8 K 2024-11-11T03:40:07,118 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c052439537498b881517d1f58de416, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731296402957 2024-11-11T03:40:07,119 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b3916ade6d4dc3988b54aeae27c718, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731296404146 2024-11-11T03:40:07,119 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686ad96e6c5949fdb0abbcd4ec4f2be0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731296404831 2024-11-11T03:40:07,120 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee387ef6fa8244289f152fa5d90b612f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731296405490 2024-11-11T03:40:07,121 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/3c54907b8f0b4bcfa1197dcadb5f60b8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/3c54907b8f0b4bcfa1197dcadb5f60b8 2024-11-11T03:40:07,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T03:40:07,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:07,123 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T03:40:07,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:07,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:07,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:07,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:07,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:07,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:07,133 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into 3c54907b8f0b4bcfa1197dcadb5f60b8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:07,133 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:07,133 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=12, startTime=1731296407027; duration=0sec 2024-11-11T03:40:07,134 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:07,134 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:07,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/eea4dc6ed0e145018ff66745502ef029 is 50, key is test_row_0/A:col10/1731296406171/Put/seqid=0 2024-11-11T03:40:07,149 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:07,149 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/d04f5393605146739238197d6ed71fe2 is 50, key is test_row_0/C:col10/1731296406123/Put/seqid=0 2024-11-11T03:40:07,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741920_1096 (size=12301) 2024-11-11T03:40:07,158 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/eea4dc6ed0e145018ff66745502ef029 2024-11-11T03:40:07,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/aacbb1263b764f8bb0f47f2d94a8f590 is 50, key is test_row_0/B:col10/1731296406171/Put/seqid=0 2024-11-11T03:40:07,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741921_1097 (size=13221) 2024-11-11T03:40:07,197 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/d04f5393605146739238197d6ed71fe2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d04f5393605146739238197d6ed71fe2 2024-11-11T03:40:07,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741922_1098 (size=12301) 2024-11-11T03:40:07,208 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into d04f5393605146739238197d6ed71fe2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:07,208 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:07,208 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/aacbb1263b764f8bb0f47f2d94a8f590 2024-11-11T03:40:07,208 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=12, startTime=1731296407027; duration=0sec 2024-11-11T03:40:07,209 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:07,209 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/56d5be35c8f847a2988754b463af1081 is 50, key is test_row_0/C:col10/1731296406171/Put/seqid=0 2024-11-11T03:40:07,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741923_1099 (size=12301) 2024-11-11T03:40:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T03:40:07,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:07,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296467339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296467340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:07,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296467451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:07,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296467452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:07,637 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/56d5be35c8f847a2988754b463af1081 2024-11-11T03:40:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/eea4dc6ed0e145018ff66745502ef029 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/eea4dc6ed0e145018ff66745502ef029 2024-11-11T03:40:07,662 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/eea4dc6ed0e145018ff66745502ef029, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T03:40:07,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/aacbb1263b764f8bb0f47f2d94a8f590 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/aacbb1263b764f8bb0f47f2d94a8f590 2024-11-11T03:40:07,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296467662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:07,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296467662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:07,670 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/aacbb1263b764f8bb0f47f2d94a8f590, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T03:40:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/56d5be35c8f847a2988754b463af1081 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/56d5be35c8f847a2988754b463af1081 2024-11-11T03:40:07,677 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/56d5be35c8f847a2988754b463af1081, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T03:40:07,678 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 488bb5013d196b74e2d5128b2d5b7358 in 555ms, sequenceid=434, compaction requested=false 2024-11-11T03:40:07,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:07,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:07,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T03:40:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T03:40:07,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T03:40:07,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4770 sec 2024-11-11T03:40:07,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4820 sec 2024-11-11T03:40:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:07,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T03:40:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:07,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:07,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0953a5e98b7144d296acc406e69248b7 is 50, key is test_row_0/A:col10/1731296407974/Put/seqid=0 2024-11-11T03:40:07,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741924_1100 (size=17181) 2024-11-11T03:40:07,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0953a5e98b7144d296acc406e69248b7 2024-11-11T03:40:08,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/4cc5dfed9b9a48228163c7ddfc16b4c4 is 50, key is test_row_0/B:col10/1731296407974/Put/seqid=0 2024-11-11T03:40:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296468015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296468019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741925_1101 (size=12301) 2024-11-11T03:40:08,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/4cc5dfed9b9a48228163c7ddfc16b4c4 2024-11-11T03:40:08,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/bfe34e042f5c4dada73579eb67805636 is 50, key is test_row_0/C:col10/1731296407974/Put/seqid=0 2024-11-11T03:40:08,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741926_1102 (size=12301) 2024-11-11T03:40:08,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/bfe34e042f5c4dada73579eb67805636 2024-11-11T03:40:08,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/0953a5e98b7144d296acc406e69248b7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0953a5e98b7144d296acc406e69248b7 2024-11-11T03:40:08,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0953a5e98b7144d296acc406e69248b7, entries=250, sequenceid=451, filesize=16.8 K 2024-11-11T03:40:08,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/4cc5dfed9b9a48228163c7ddfc16b4c4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/4cc5dfed9b9a48228163c7ddfc16b4c4 2024-11-11T03:40:08,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/4cc5dfed9b9a48228163c7ddfc16b4c4, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T03:40:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/bfe34e042f5c4dada73579eb67805636 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bfe34e042f5c4dada73579eb67805636 2024-11-11T03:40:08,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bfe34e042f5c4dada73579eb67805636, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T03:40:08,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 488bb5013d196b74e2d5128b2d5b7358 in 108ms, sequenceid=451, compaction requested=true 2024-11-11T03:40:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:08,084 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:08,084 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:08,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:08,085 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:08,085 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:08,085 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:40:08,085 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:40:08,085 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,085 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,085 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f0f7639a0ea42f9b45745661d8b5820, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/eea4dc6ed0e145018ff66745502ef029, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0953a5e98b7144d296acc406e69248b7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=41.7 K 2024-11-11T03:40:08,085 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/3c54907b8f0b4bcfa1197dcadb5f60b8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/aacbb1263b764f8bb0f47f2d94a8f590, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/4cc5dfed9b9a48228163c7ddfc16b4c4] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=36.9 K 2024-11-11T03:40:08,086 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c54907b8f0b4bcfa1197dcadb5f60b8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731296405490 2024-11-11T03:40:08,086 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0f7639a0ea42f9b45745661d8b5820, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731296405490 2024-11-11T03:40:08,086 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea4dc6ed0e145018ff66745502ef029, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731296406160 2024-11-11T03:40:08,086 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting aacbb1263b764f8bb0f47f2d94a8f590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731296406160 2024-11-11T03:40:08,087 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc5dfed9b9a48228163c7ddfc16b4c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731296407337 2024-11-11T03:40:08,087 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0953a5e98b7144d296acc406e69248b7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731296407332 2024-11-11T03:40:08,096 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:08,097 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/313d14c4df1440eaa2792580a5750a65 is 50, key is test_row_0/B:col10/1731296407974/Put/seqid=0 2024-11-11T03:40:08,101 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:08,101 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/33aacc858b514314a0cc58b9b028f3e2 is 50, key is test_row_0/A:col10/1731296407974/Put/seqid=0 2024-11-11T03:40:08,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741927_1103 (size=13323) 2024-11-11T03:40:08,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741928_1104 (size=13323) 2024-11-11T03:40:08,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T03:40:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:08,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:08,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/883b2e3098ec46008350f161f2871b3e is 50, key is test_row_0/A:col10/1731296408136/Put/seqid=0 2024-11-11T03:40:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296468175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296468180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741929_1105 (size=17181) 2024-11-11T03:40:08,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/883b2e3098ec46008350f161f2871b3e 2024-11-11T03:40:08,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/453acc9560d9448aa80a6f3309060af2 is 50, key is test_row_0/B:col10/1731296408136/Put/seqid=0 2024-11-11T03:40:08,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741930_1106 (size=12301) 2024-11-11T03:40:08,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/453acc9560d9448aa80a6f3309060af2 2024-11-11T03:40:08,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/728dd183a76742638ed7eaab1f76f4ae is 50, key is test_row_0/C:col10/1731296408136/Put/seqid=0 2024-11-11T03:40:08,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296468281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741931_1107 (size=12301) 2024-11-11T03:40:08,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/728dd183a76742638ed7eaab1f76f4ae 2024-11-11T03:40:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296468292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/883b2e3098ec46008350f161f2871b3e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/883b2e3098ec46008350f161f2871b3e 2024-11-11T03:40:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/883b2e3098ec46008350f161f2871b3e, entries=250, sequenceid=470, filesize=16.8 K 2024-11-11T03:40:08,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/453acc9560d9448aa80a6f3309060af2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/453acc9560d9448aa80a6f3309060af2 2024-11-11T03:40:08,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/453acc9560d9448aa80a6f3309060af2, entries=150, sequenceid=470, filesize=12.0 K 2024-11-11T03:40:08,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/728dd183a76742638ed7eaab1f76f4ae as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/728dd183a76742638ed7eaab1f76f4ae 2024-11-11T03:40:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T03:40:08,352 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:08,352 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:08,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/728dd183a76742638ed7eaab1f76f4ae, entries=150, sequenceid=470, filesize=12.0 K 2024-11-11T03:40:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T03:40:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T03:40:08,355 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:08,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:08,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:08,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 488bb5013d196b74e2d5128b2d5b7358 in 220ms, sequenceid=470, compaction requested=true 2024-11-11T03:40:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:08,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T03:40:08,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:08,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T03:40:08,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:08,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-11T03:40:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T03:40:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:08,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T03:40:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:08,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/a20f1e2a536a4e2397626da5f147b8fb is 50, key is test_row_0/A:col10/1731296408166/Put/seqid=0 2024-11-11T03:40:08,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741932_1108 (size=14741) 2024-11-11T03:40:08,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T03:40:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,534 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/313d14c4df1440eaa2792580a5750a65 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/313d14c4df1440eaa2792580a5750a65 2024-11-11T03:40:08,541 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/33aacc858b514314a0cc58b9b028f3e2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/33aacc858b514314a0cc58b9b028f3e2 2024-11-11T03:40:08,548 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into 313d14c4df1440eaa2792580a5750a65(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:08,548 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:08,552 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 33aacc858b514314a0cc58b9b028f3e2(size=13.0 K), total size for store is 29.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:08,552 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:08,552 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296408084; duration=0sec 2024-11-11T03:40:08,553 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-11T03:40:08,553 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:08,553 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:08,553 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:08,553 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296408084; duration=0sec 2024-11-11T03:40:08,553 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T03:40:08,553 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:08,553 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:08,555 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:08,555 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:40:08,555 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,555 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d04f5393605146739238197d6ed71fe2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/56d5be35c8f847a2988754b463af1081, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bfe34e042f5c4dada73579eb67805636, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/728dd183a76742638ed7eaab1f76f4ae] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=48.9 K 2024-11-11T03:40:08,556 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting d04f5393605146739238197d6ed71fe2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731296405490 2024-11-11T03:40:08,556 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-11T03:40:08,556 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T03:40:08,556 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56d5be35c8f847a2988754b463af1081, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731296406160 2024-11-11T03:40:08,557 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe34e042f5c4dada73579eb67805636, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731296407337 2024-11-11T03:40:08,557 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 728dd183a76742638ed7eaab1f76f4ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731296408017 2024-11-11T03:40:08,557 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T03:40:08,557 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. because compaction request was cancelled 2024-11-11T03:40:08,557 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:08,558 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T03:40:08,559 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T03:40:08,559 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T03:40:08,559 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. because compaction request was cancelled 2024-11-11T03:40:08,559 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:08,562 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T03:40:08,563 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T03:40:08,563 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T03:40:08,563 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. because compaction request was cancelled 2024-11-11T03:40:08,563 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296468557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,572 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:08,573 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/5a9a6302452f4872a9890f021ea2e16b is 50, key is test_row_0/C:col10/1731296408136/Put/seqid=0 2024-11-11T03:40:08,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296468569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741933_1109 (size=13357) 2024-11-11T03:40:08,589 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/5a9a6302452f4872a9890f021ea2e16b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/5a9a6302452f4872a9890f021ea2e16b 2024-11-11T03:40:08,596 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into 5a9a6302452f4872a9890f021ea2e16b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:08,596 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:08,596 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=12, startTime=1731296408359; duration=0sec 2024-11-11T03:40:08,596 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:08,596 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T03:40:08,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T03:40:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296468672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296468682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T03:40:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296468881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296468892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:08,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/a20f1e2a536a4e2397626da5f147b8fb 2024-11-11T03:40:08,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/dc70577c555f4b29a74aadaa15197ecf is 50, key is test_row_0/B:col10/1731296408166/Put/seqid=0 2024-11-11T03:40:08,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741934_1110 (size=12301) 2024-11-11T03:40:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T03:40:08,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T03:40:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:08,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:09,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T03:40:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:09,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296469192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:09,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296469211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:09,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T03:40:09,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:09,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/dc70577c555f4b29a74aadaa15197ecf 2024-11-11T03:40:09,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/dcb9064658824e6e9ce61bc2dc32ad0c is 50, key is test_row_0/C:col10/1731296408166/Put/seqid=0 2024-11-11T03:40:09,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741935_1111 (size=12301) 2024-11-11T03:40:09,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/dcb9064658824e6e9ce61bc2dc32ad0c 2024-11-11T03:40:09,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/a20f1e2a536a4e2397626da5f147b8fb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a20f1e2a536a4e2397626da5f147b8fb 2024-11-11T03:40:09,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a20f1e2a536a4e2397626da5f147b8fb, entries=200, sequenceid=484, filesize=14.4 K 2024-11-11T03:40:09,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/dc70577c555f4b29a74aadaa15197ecf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/dc70577c555f4b29a74aadaa15197ecf 2024-11-11T03:40:09,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/dc70577c555f4b29a74aadaa15197ecf, entries=150, sequenceid=484, filesize=12.0 K 2024-11-11T03:40:09,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/dcb9064658824e6e9ce61bc2dc32ad0c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/dcb9064658824e6e9ce61bc2dc32ad0c 2024-11-11T03:40:09,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/dcb9064658824e6e9ce61bc2dc32ad0c, entries=150, sequenceid=484, filesize=12.0 K 2024-11-11T03:40:09,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 488bb5013d196b74e2d5128b2d5b7358 in 891ms, sequenceid=484, compaction requested=true 2024-11-11T03:40:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:09,387 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:09,387 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:09,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:09,390 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:09,390 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:40:09,390 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,390 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/313d14c4df1440eaa2792580a5750a65, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/453acc9560d9448aa80a6f3309060af2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/dc70577c555f4b29a74aadaa15197ecf] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=37.0 K 2024-11-11T03:40:09,390 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:09,390 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:40:09,390 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,391 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/33aacc858b514314a0cc58b9b028f3e2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/883b2e3098ec46008350f161f2871b3e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a20f1e2a536a4e2397626da5f147b8fb] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=44.2 K 2024-11-11T03:40:09,391 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 313d14c4df1440eaa2792580a5750a65, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731296407337 2024-11-11T03:40:09,391 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33aacc858b514314a0cc58b9b028f3e2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731296407337 2024-11-11T03:40:09,391 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 453acc9560d9448aa80a6f3309060af2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731296408017 2024-11-11T03:40:09,392 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 883b2e3098ec46008350f161f2871b3e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731296408001 2024-11-11T03:40:09,392 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting dc70577c555f4b29a74aadaa15197ecf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731296408166 2024-11-11T03:40:09,392 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a20f1e2a536a4e2397626da5f147b8fb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731296408156 2024-11-11T03:40:09,405 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:09,406 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/c86d1844801f45eba6e9550cd8bf561f is 50, key is test_row_0/A:col10/1731296408166/Put/seqid=0 2024-11-11T03:40:09,409 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:09,409 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/c63b47703d194e659940cfa04e0dff52 is 50, key is test_row_0/B:col10/1731296408166/Put/seqid=0 2024-11-11T03:40:09,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741936_1112 (size=13425) 2024-11-11T03:40:09,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741937_1113 (size=13425) 2024-11-11T03:40:09,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T03:40:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,444 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T03:40:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:09,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/b784fffaea0e4f389b5e07d01ebe268a is 50, key is test_row_0/A:col10/1731296408553/Put/seqid=0 2024-11-11T03:40:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741938_1114 (size=12301) 2024-11-11T03:40:09,491 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/b784fffaea0e4f389b5e07d01ebe268a 2024-11-11T03:40:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T03:40:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/ccb78ab7b1e24af89f952fbd173568bb is 50, key is test_row_0/B:col10/1731296408553/Put/seqid=0 2024-11-11T03:40:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741939_1115 (size=12301) 2024-11-11T03:40:09,529 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/ccb78ab7b1e24af89f952fbd173568bb 2024-11-11T03:40:09,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/deaf6e92ba2b4221bbb675aee8148544 is 50, key is test_row_0/C:col10/1731296408553/Put/seqid=0 2024-11-11T03:40:09,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741940_1116 (size=12301) 2024-11-11T03:40:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. as already flushing 2024-11-11T03:40:09,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296469763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:09,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296469767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:09,837 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/c86d1844801f45eba6e9550cd8bf561f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/c86d1844801f45eba6e9550cd8bf561f 2024-11-11T03:40:09,839 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/c63b47703d194e659940cfa04e0dff52 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/c63b47703d194e659940cfa04e0dff52 2024-11-11T03:40:09,848 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into c63b47703d194e659940cfa04e0dff52(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:09,848 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:09,848 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296409387; duration=0sec 2024-11-11T03:40:09,848 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:09,848 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:09,848 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T03:40:09,849 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T03:40:09,850 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T03:40:09,850 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. because compaction request was cancelled 2024-11-11T03:40:09,850 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:09,851 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into c86d1844801f45eba6e9550cd8bf561f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:09,851 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:09,851 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296409387; duration=0sec 2024-11-11T03:40:09,851 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:09,851 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:09,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48172 deadline: 1731296469872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:09,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731296469872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:09,954 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/deaf6e92ba2b4221bbb675aee8148544 2024-11-11T03:40:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/b784fffaea0e4f389b5e07d01ebe268a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b784fffaea0e4f389b5e07d01ebe268a 2024-11-11T03:40:09,968 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b784fffaea0e4f389b5e07d01ebe268a, entries=150, sequenceid=505, filesize=12.0 K 2024-11-11T03:40:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/ccb78ab7b1e24af89f952fbd173568bb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ccb78ab7b1e24af89f952fbd173568bb 2024-11-11T03:40:09,975 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ccb78ab7b1e24af89f952fbd173568bb, entries=150, sequenceid=505, filesize=12.0 K 2024-11-11T03:40:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/deaf6e92ba2b4221bbb675aee8148544 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/deaf6e92ba2b4221bbb675aee8148544 2024-11-11T03:40:09,982 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/deaf6e92ba2b4221bbb675aee8148544, entries=150, sequenceid=505, filesize=12.0 K 2024-11-11T03:40:09,983 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 488bb5013d196b74e2d5128b2d5b7358 in 539ms, sequenceid=505, compaction requested=true 2024-11-11T03:40:09,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:09,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:09,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T03:40:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T03:40:09,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T03:40:09,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-11T03:40:09,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6350 sec 2024-11-11T03:40:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:10,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:40:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:10,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/57562ac643d544c1b702e3c629db97ce is 50, key is test_row_0/A:col10/1731296409765/Put/seqid=0 2024-11-11T03:40:10,118 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T03:40:10,118 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T03:40:10,118 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:10,118 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:10,119 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T03:40:10,119 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:10,123 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T03:40:10,123 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:10,123 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,123 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,124 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,124 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:10,125 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,125 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,125 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,125 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:10,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:10,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741941_1117 (size=17181) 2024-11-11T03:40:10,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:10,127 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T03:40:10,127 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:10,127 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,127 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:10,135 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T03:40:10,135 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:10,135 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,136 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:10,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T03:40:10,512 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:10,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/57562ac643d544c1b702e3c629db97ce 2024-11-11T03:40:10,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/e27f599fac6e47018b0c5c6cf3b7103f is 50, key is test_row_0/B:col10/1731296409765/Put/seqid=0 2024-11-11T03:40:10,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741942_1118 (size=12301) 2024-11-11T03:40:10,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/e27f599fac6e47018b0c5c6cf3b7103f 2024-11-11T03:40:10,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/eed49c8ded2c4b48ab7ad59dc155d704 is 50, key is test_row_0/C:col10/1731296409765/Put/seqid=0 2024-11-11T03:40:10,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741943_1119 (size=12301) 2024-11-11T03:40:11,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/eed49c8ded2c4b48ab7ad59dc155d704 2024-11-11T03:40:11,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/57562ac643d544c1b702e3c629db97ce as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/57562ac643d544c1b702e3c629db97ce 2024-11-11T03:40:11,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/57562ac643d544c1b702e3c629db97ce, entries=250, sequenceid=523, filesize=16.8 K 2024-11-11T03:40:11,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/e27f599fac6e47018b0c5c6cf3b7103f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e27f599fac6e47018b0c5c6cf3b7103f 2024-11-11T03:40:11,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e27f599fac6e47018b0c5c6cf3b7103f, entries=150, sequenceid=523, filesize=12.0 K 2024-11-11T03:40:11,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/eed49c8ded2c4b48ab7ad59dc155d704 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eed49c8ded2c4b48ab7ad59dc155d704 2024-11-11T03:40:11,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eed49c8ded2c4b48ab7ad59dc155d704, entries=150, sequenceid=523, filesize=12.0 K 2024-11-11T03:40:11,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=60.38 KB/61830 for 488bb5013d196b74e2d5128b2d5b7358 in 1295ms, sequenceid=523, compaction requested=true 2024-11-11T03:40:11,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:11,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:11,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:11,394 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:11,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:11,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:11,394 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:11,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 488bb5013d196b74e2d5128b2d5b7358:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:11,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:11,395 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:11,395 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:11,395 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/A is initiating minor compaction (all files) 2024-11-11T03:40:11,395 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/B is initiating minor compaction (all files) 2024-11-11T03:40:11,395 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/A in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:11,395 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/B in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:11,395 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/c86d1844801f45eba6e9550cd8bf561f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b784fffaea0e4f389b5e07d01ebe268a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/57562ac643d544c1b702e3c629db97ce] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=41.9 K 2024-11-11T03:40:11,395 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/c63b47703d194e659940cfa04e0dff52, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ccb78ab7b1e24af89f952fbd173568bb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e27f599fac6e47018b0c5c6cf3b7103f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=37.1 K 2024-11-11T03:40:11,396 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86d1844801f45eba6e9550cd8bf561f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731296408166 2024-11-11T03:40:11,396 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting c63b47703d194e659940cfa04e0dff52, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731296408166 2024-11-11T03:40:11,396 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ccb78ab7b1e24af89f952fbd173568bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731296408526 2024-11-11T03:40:11,396 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting b784fffaea0e4f389b5e07d01ebe268a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731296408526 2024-11-11T03:40:11,396 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e27f599fac6e47018b0c5c6cf3b7103f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731296409753 2024-11-11T03:40:11,396 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57562ac643d544c1b702e3c629db97ce, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731296409753 2024-11-11T03:40:11,405 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#B#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:11,405 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a4a00b546ebc4103b37842977ed658a5 is 50, key is test_row_0/B:col10/1731296409765/Put/seqid=0 2024-11-11T03:40:11,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741944_1120 (size=13527) 2024-11-11T03:40:11,427 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#A#compaction#108 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:11,428 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/263091133cea492c82fb926cbe035560 is 50, key is test_row_0/A:col10/1731296409765/Put/seqid=0 2024-11-11T03:40:11,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741945_1121 (size=13527) 2024-11-11T03:40:11,820 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/a4a00b546ebc4103b37842977ed658a5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a4a00b546ebc4103b37842977ed658a5 2024-11-11T03:40:11,829 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/B of 488bb5013d196b74e2d5128b2d5b7358 into a4a00b546ebc4103b37842977ed658a5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:11,829 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:11,829 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/B, priority=13, startTime=1731296411394; duration=0sec 2024-11-11T03:40:11,829 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:11,829 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:B 2024-11-11T03:40:11,830 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:11,832 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:11,832 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 488bb5013d196b74e2d5128b2d5b7358/C is initiating minor compaction (all files) 2024-11-11T03:40:11,832 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 488bb5013d196b74e2d5128b2d5b7358/C in TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:11,832 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/5a9a6302452f4872a9890f021ea2e16b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/dcb9064658824e6e9ce61bc2dc32ad0c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/deaf6e92ba2b4221bbb675aee8148544, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eed49c8ded2c4b48ab7ad59dc155d704] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp, totalSize=49.1 K 2024-11-11T03:40:11,833 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9a6302452f4872a9890f021ea2e16b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731296408017 2024-11-11T03:40:11,833 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb9064658824e6e9ce61bc2dc32ad0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731296408166 2024-11-11T03:40:11,834 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting deaf6e92ba2b4221bbb675aee8148544, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731296408526 2024-11-11T03:40:11,834 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting eed49c8ded2c4b48ab7ad59dc155d704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731296409753 2024-11-11T03:40:11,838 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/263091133cea492c82fb926cbe035560 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/263091133cea492c82fb926cbe035560 2024-11-11T03:40:11,845 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/A of 488bb5013d196b74e2d5128b2d5b7358 into 263091133cea492c82fb926cbe035560(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:11,845 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:11,845 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/A, priority=13, startTime=1731296411394; duration=0sec 2024-11-11T03:40:11,845 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:11,845 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:A 2024-11-11T03:40:11,849 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 488bb5013d196b74e2d5128b2d5b7358#C#compaction#109 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:11,850 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/b46ed14e2fa749cdb2e66247ede6df77 is 50, key is test_row_0/C:col10/1731296409765/Put/seqid=0 2024-11-11T03:40:11,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741946_1122 (size=13493) 2024-11-11T03:40:12,260 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/b46ed14e2fa749cdb2e66247ede6df77 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b46ed14e2fa749cdb2e66247ede6df77 2024-11-11T03:40:12,265 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 488bb5013d196b74e2d5128b2d5b7358/C of 488bb5013d196b74e2d5128b2d5b7358 into b46ed14e2fa749cdb2e66247ede6df77(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:12,266 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:12,266 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., storeName=488bb5013d196b74e2d5128b2d5b7358/C, priority=12, startTime=1731296411394; duration=0sec 2024-11-11T03:40:12,266 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:12,266 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 488bb5013d196b74e2d5128b2d5b7358:C 2024-11-11T03:40:12,621 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:40:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:14,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T03:40:14,526 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T03:40:14,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:14,526 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:14,526 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:14,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:14,527 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:14,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:14,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:14,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:14,534 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T03:40:14,534 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:14,534 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,535 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:14,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/158067a3d8fa481f929f83aa34adb8f9 is 50, key is test_row_0/A:col10/1731296410114/Put/seqid=0 2024-11-11T03:40:14,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741947_1123 (size=12301) 2024-11-11T03:40:14,574 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T03:40:14,574 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:14,574 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,574 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2495 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2513 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T03:40:14,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1148 2024-11-11T03:40:14,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3441 rows 2024-11-11T03:40:14,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1133 2024-11-11T03:40:14,575 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3393 rows 2024-11-11T03:40:14,575 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:40:14,575 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:40:14,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:14,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:14,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T03:40:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T03:40:14,587 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296414586"}]},"ts":"1731296414586"} 2024-11-11T03:40:14,589 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T03:40:14,589 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T03:40:14,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:40:14,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, UNASSIGN}] 2024-11-11T03:40:14,596 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, UNASSIGN 2024-11-11T03:40:14,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=488bb5013d196b74e2d5128b2d5b7358, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:14,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, UNASSIGN because future has completed 2024-11-11T03:40:14,600 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:40:14,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:40:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T03:40:14,757 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:14,757 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:40:14,758 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 488bb5013d196b74e2d5128b2d5b7358, disabling compactions & flushes 2024-11-11T03:40:14,758 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T03:40:14,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/158067a3d8fa481f929f83aa34adb8f9 2024-11-11T03:40:14,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/df6483beafd14e3e882d7ef0f1689c15 is 50, key is test_row_0/B:col10/1731296410114/Put/seqid=0 2024-11-11T03:40:14,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741948_1124 (size=12301) 2024-11-11T03:40:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T03:40:15,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/df6483beafd14e3e882d7ef0f1689c15 2024-11-11T03:40:15,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ab722af8eff54810820033e08c9cf206 is 50, key is test_row_0/C:col10/1731296410114/Put/seqid=0 2024-11-11T03:40:15,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741949_1125 (size=12301) 2024-11-11T03:40:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T03:40:15,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ab722af8eff54810820033e08c9cf206 2024-11-11T03:40:15,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/158067a3d8fa481f929f83aa34adb8f9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/158067a3d8fa481f929f83aa34adb8f9 2024-11-11T03:40:15,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/158067a3d8fa481f929f83aa34adb8f9, entries=150, sequenceid=539, filesize=12.0 K 2024-11-11T03:40:15,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/df6483beafd14e3e882d7ef0f1689c15 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/df6483beafd14e3e882d7ef0f1689c15 2024-11-11T03:40:15,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/df6483beafd14e3e882d7ef0f1689c15, entries=150, sequenceid=539, filesize=12.0 K 2024-11-11T03:40:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/ab722af8eff54810820033e08c9cf206 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ab722af8eff54810820033e08c9cf206 2024-11-11T03:40:15,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ab722af8eff54810820033e08c9cf206, entries=150, sequenceid=539, filesize=12.0 K 2024-11-11T03:40:15,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=13.42 KB/13740 for 488bb5013d196b74e2d5128b2d5b7358 in 1291ms, sequenceid=539, compaction requested=false 2024-11-11T03:40:15,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 488bb5013d196b74e2d5128b2d5b7358: 2024-11-11T03:40:15,816 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:15,816 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:15,816 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. after waiting 0 ms 2024-11-11T03:40:15,816 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:15,816 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 488bb5013d196b74e2d5128b2d5b7358 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-11T03:40:15,816 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=A 2024-11-11T03:40:15,816 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:15,817 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=B 2024-11-11T03:40:15,817 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:15,817 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 488bb5013d196b74e2d5128b2d5b7358, store=C 2024-11-11T03:40:15,817 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:15,820 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8e3dafc324bd43158d948fa0fddc8fe6 is 50, key is test_row_0/A:col10/1731296414572/Put/seqid=0 2024-11-11T03:40:15,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741950_1126 (size=7415) 2024-11-11T03:40:16,225 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8e3dafc324bd43158d948fa0fddc8fe6 2024-11-11T03:40:16,234 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2ea61679d083489e847924e031d0a3f5 is 50, key is test_row_0/B:col10/1731296414572/Put/seqid=0 2024-11-11T03:40:16,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741951_1127 (size=7415) 2024-11-11T03:40:16,639 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2ea61679d083489e847924e031d0a3f5 2024-11-11T03:40:16,655 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f1ff77fdfea04111b6dc244a828ee02d is 50, key is test_row_0/C:col10/1731296414572/Put/seqid=0 2024-11-11T03:40:16,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741952_1128 (size=7415) 2024-11-11T03:40:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T03:40:17,061 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f1ff77fdfea04111b6dc244a828ee02d 2024-11-11T03:40:17,066 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/A/8e3dafc324bd43158d948fa0fddc8fe6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8e3dafc324bd43158d948fa0fddc8fe6 2024-11-11T03:40:17,071 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8e3dafc324bd43158d948fa0fddc8fe6, entries=50, sequenceid=544, filesize=7.2 K 2024-11-11T03:40:17,072 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/B/2ea61679d083489e847924e031d0a3f5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2ea61679d083489e847924e031d0a3f5 2024-11-11T03:40:17,076 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2ea61679d083489e847924e031d0a3f5, entries=50, sequenceid=544, filesize=7.2 K 2024-11-11T03:40:17,077 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/.tmp/C/f1ff77fdfea04111b6dc244a828ee02d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1ff77fdfea04111b6dc244a828ee02d 2024-11-11T03:40:17,081 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1ff77fdfea04111b6dc244a828ee02d, entries=50, sequenceid=544, filesize=7.2 K 2024-11-11T03:40:17,082 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 488bb5013d196b74e2d5128b2d5b7358 in 1266ms, sequenceid=544, compaction requested=true 2024-11-11T03:40:17,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/603c522a70c0454f980b567bae66a7b9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a28329239071493d82e698a23888d235, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/50e07a830c4b43a8b3cb5a01d1832645, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4b1e9ac7e5b14ced97c150fd4685c99c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/76efcbccc14b463a9a3a911251cb1005, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/5e6ca550b6d24e27b13d9a7e338b8762, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b3923fc62e9b4eb59b8df31df024648d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/86f3da9f35fb4beda39c0ff82234d1f6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f689ed6102f46b8b20eb8db14c93db6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/10efa001273b4be2ac705200ba7808c1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/ca8c8f52c31941e7ad4a70e158b20eb1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/05735f1b06694868a61bb581048be9b5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9359bc45d8d9440ba44a571ead594092, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9bb241c292ea465395ee4da60fccb1b5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/1ee6bd3de5a5424d8fc7ec8c7239e2de, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/879509d3938949f6ad0c1a1ae4db0a67, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0f1cd61eb5e54b0fbbf42abeeb14b83f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0e2b3a6374154f0da882087258701af4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/e059286810ec42e892e05b48dc845e12, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/2a17d1bf16d546e898b7e47e005a8d6b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/58beb69e9a3a40a1b5bdadd0d35d27c8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/6245b0b1139846758f84e5551f1ce41e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/128872c0b26942fabae2e09302435c43, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4e28477cfe1d4b11b72ef272bd139dad, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/870ee0d5553d4822b268fcbe04a59665, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/7385477af4324ac9abbd8d82a4b6c578, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f0f7639a0ea42f9b45745661d8b5820, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9eecae8310bd4f06aa0816ab771307e6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/eea4dc6ed0e145018ff66745502ef029, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0953a5e98b7144d296acc406e69248b7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/33aacc858b514314a0cc58b9b028f3e2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/883b2e3098ec46008350f161f2871b3e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a20f1e2a536a4e2397626da5f147b8fb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/c86d1844801f45eba6e9550cd8bf561f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b784fffaea0e4f389b5e07d01ebe268a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/57562ac643d544c1b702e3c629db97ce] to archive 2024-11-11T03:40:17,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:40:17,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/603c522a70c0454f980b567bae66a7b9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/603c522a70c0454f980b567bae66a7b9 2024-11-11T03:40:17,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a28329239071493d82e698a23888d235 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a28329239071493d82e698a23888d235 2024-11-11T03:40:17,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/50e07a830c4b43a8b3cb5a01d1832645 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/50e07a830c4b43a8b3cb5a01d1832645 2024-11-11T03:40:17,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4b1e9ac7e5b14ced97c150fd4685c99c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4b1e9ac7e5b14ced97c150fd4685c99c 2024-11-11T03:40:17,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/76efcbccc14b463a9a3a911251cb1005 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/76efcbccc14b463a9a3a911251cb1005 2024-11-11T03:40:17,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/5e6ca550b6d24e27b13d9a7e338b8762 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/5e6ca550b6d24e27b13d9a7e338b8762 2024-11-11T03:40:17,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b3923fc62e9b4eb59b8df31df024648d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b3923fc62e9b4eb59b8df31df024648d 2024-11-11T03:40:17,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/86f3da9f35fb4beda39c0ff82234d1f6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/86f3da9f35fb4beda39c0ff82234d1f6 2024-11-11T03:40:17,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f689ed6102f46b8b20eb8db14c93db6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f689ed6102f46b8b20eb8db14c93db6 2024-11-11T03:40:17,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/10efa001273b4be2ac705200ba7808c1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/10efa001273b4be2ac705200ba7808c1 2024-11-11T03:40:17,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/ca8c8f52c31941e7ad4a70e158b20eb1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/ca8c8f52c31941e7ad4a70e158b20eb1 2024-11-11T03:40:17,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/05735f1b06694868a61bb581048be9b5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/05735f1b06694868a61bb581048be9b5 2024-11-11T03:40:17,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9359bc45d8d9440ba44a571ead594092 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9359bc45d8d9440ba44a571ead594092 2024-11-11T03:40:17,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9bb241c292ea465395ee4da60fccb1b5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9bb241c292ea465395ee4da60fccb1b5 2024-11-11T03:40:17,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/1ee6bd3de5a5424d8fc7ec8c7239e2de to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/1ee6bd3de5a5424d8fc7ec8c7239e2de 2024-11-11T03:40:17,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/879509d3938949f6ad0c1a1ae4db0a67 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/879509d3938949f6ad0c1a1ae4db0a67 2024-11-11T03:40:17,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0f1cd61eb5e54b0fbbf42abeeb14b83f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0f1cd61eb5e54b0fbbf42abeeb14b83f 2024-11-11T03:40:17,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0e2b3a6374154f0da882087258701af4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0e2b3a6374154f0da882087258701af4 2024-11-11T03:40:17,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/e059286810ec42e892e05b48dc845e12 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/e059286810ec42e892e05b48dc845e12 2024-11-11T03:40:17,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/2a17d1bf16d546e898b7e47e005a8d6b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/2a17d1bf16d546e898b7e47e005a8d6b 2024-11-11T03:40:17,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/58beb69e9a3a40a1b5bdadd0d35d27c8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/58beb69e9a3a40a1b5bdadd0d35d27c8 2024-11-11T03:40:17,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/6245b0b1139846758f84e5551f1ce41e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/6245b0b1139846758f84e5551f1ce41e 2024-11-11T03:40:17,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/128872c0b26942fabae2e09302435c43 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/128872c0b26942fabae2e09302435c43 2024-11-11T03:40:17,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4e28477cfe1d4b11b72ef272bd139dad to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/4e28477cfe1d4b11b72ef272bd139dad 2024-11-11T03:40:17,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/870ee0d5553d4822b268fcbe04a59665 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/870ee0d5553d4822b268fcbe04a59665 2024-11-11T03:40:17,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/7385477af4324ac9abbd8d82a4b6c578 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/7385477af4324ac9abbd8d82a4b6c578 2024-11-11T03:40:17,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f0f7639a0ea42f9b45745661d8b5820 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8f0f7639a0ea42f9b45745661d8b5820 2024-11-11T03:40:17,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9eecae8310bd4f06aa0816ab771307e6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/9eecae8310bd4f06aa0816ab771307e6 2024-11-11T03:40:17,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/eea4dc6ed0e145018ff66745502ef029 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/eea4dc6ed0e145018ff66745502ef029 2024-11-11T03:40:17,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0953a5e98b7144d296acc406e69248b7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/0953a5e98b7144d296acc406e69248b7 2024-11-11T03:40:17,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/33aacc858b514314a0cc58b9b028f3e2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/33aacc858b514314a0cc58b9b028f3e2 2024-11-11T03:40:17,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/883b2e3098ec46008350f161f2871b3e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/883b2e3098ec46008350f161f2871b3e 2024-11-11T03:40:17,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a20f1e2a536a4e2397626da5f147b8fb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/a20f1e2a536a4e2397626da5f147b8fb 2024-11-11T03:40:17,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/c86d1844801f45eba6e9550cd8bf561f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/c86d1844801f45eba6e9550cd8bf561f 2024-11-11T03:40:17,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b784fffaea0e4f389b5e07d01ebe268a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/b784fffaea0e4f389b5e07d01ebe268a 2024-11-11T03:40:17,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/57562ac643d544c1b702e3c629db97ce to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/57562ac643d544c1b702e3c629db97ce 2024-11-11T03:40:17,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/0910b28ff7dc4c229d00754540d764e8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/61bfb881ee374328a668c4fee4d3d72d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e0e0f9f75d4a49ccb089caa884b1a1c3, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8b571cfd332c4407ad74c7046126c90e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2abfc1deefa5495a8b4f8fbb535953ec, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2a033dc22b314d8cb68a2de4d5c63d81, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/386d36316a544a468997bdc82cae994b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/5e1260b511ff45f5b3e4398eebd0dc98, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/cace7fcb2b784a6485a7e01b1ac847ce, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/b3a8735058fd469a91a3afff52fd425f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a2c48f06b0e4447b84a5e40bdf65bdd8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/caeeb00c41594fa89c8c45d697c75fd8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8acd56e6f6414f50b89c0a17a977fd5f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/1426810f812242e6bf5a8d2b9b6f3f4a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a080a2d2c79a4ca9af6676b84b48408f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/bbdf53628eae41d9a6c71de8fd70954a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f6508cd46f5d42c380834ba015c5e662, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/51f626f1635f44949d115db3fcca6cd8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/41ff48bef1894c8ca696d77d25b52582, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/07355ab769f44ec78cce2b7757485ed4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/10c8466b898343cb9de2896611c5b5e2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/388b2a4fdcac48f3bdb90648b5f5f1c0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/407c2bc62f8e4f9f880eed7e66e5f450, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/9a735f880da8427a916073c11ffb24aa, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f64846bf826f4daeb5d13335b3569a8b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/615d642e511a42ea83e403182ae905c6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/3c54907b8f0b4bcfa1197dcadb5f60b8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ce1a23298b114a25b12eeefcbd2b9c92, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/aacbb1263b764f8bb0f47f2d94a8f590, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/313d14c4df1440eaa2792580a5750a65, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/4cc5dfed9b9a48228163c7ddfc16b4c4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/453acc9560d9448aa80a6f3309060af2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/c63b47703d194e659940cfa04e0dff52, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/dc70577c555f4b29a74aadaa15197ecf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ccb78ab7b1e24af89f952fbd173568bb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e27f599fac6e47018b0c5c6cf3b7103f] to archive 2024-11-11T03:40:17,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:40:17,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/0910b28ff7dc4c229d00754540d764e8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/0910b28ff7dc4c229d00754540d764e8 2024-11-11T03:40:17,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/61bfb881ee374328a668c4fee4d3d72d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/61bfb881ee374328a668c4fee4d3d72d 2024-11-11T03:40:17,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e0e0f9f75d4a49ccb089caa884b1a1c3 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e0e0f9f75d4a49ccb089caa884b1a1c3 2024-11-11T03:40:17,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8b571cfd332c4407ad74c7046126c90e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8b571cfd332c4407ad74c7046126c90e 2024-11-11T03:40:17,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2abfc1deefa5495a8b4f8fbb535953ec to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2abfc1deefa5495a8b4f8fbb535953ec 2024-11-11T03:40:17,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2a033dc22b314d8cb68a2de4d5c63d81 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2a033dc22b314d8cb68a2de4d5c63d81 2024-11-11T03:40:17,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/386d36316a544a468997bdc82cae994b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/386d36316a544a468997bdc82cae994b 2024-11-11T03:40:17,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/5e1260b511ff45f5b3e4398eebd0dc98 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/5e1260b511ff45f5b3e4398eebd0dc98 2024-11-11T03:40:17,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/cace7fcb2b784a6485a7e01b1ac847ce to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/cace7fcb2b784a6485a7e01b1ac847ce 2024-11-11T03:40:17,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/b3a8735058fd469a91a3afff52fd425f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/b3a8735058fd469a91a3afff52fd425f 2024-11-11T03:40:17,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a2c48f06b0e4447b84a5e40bdf65bdd8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a2c48f06b0e4447b84a5e40bdf65bdd8 2024-11-11T03:40:17,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/caeeb00c41594fa89c8c45d697c75fd8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/caeeb00c41594fa89c8c45d697c75fd8 2024-11-11T03:40:17,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8acd56e6f6414f50b89c0a17a977fd5f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/8acd56e6f6414f50b89c0a17a977fd5f 2024-11-11T03:40:17,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/1426810f812242e6bf5a8d2b9b6f3f4a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/1426810f812242e6bf5a8d2b9b6f3f4a 2024-11-11T03:40:17,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a080a2d2c79a4ca9af6676b84b48408f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a080a2d2c79a4ca9af6676b84b48408f 2024-11-11T03:40:17,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/bbdf53628eae41d9a6c71de8fd70954a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/bbdf53628eae41d9a6c71de8fd70954a 2024-11-11T03:40:17,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f6508cd46f5d42c380834ba015c5e662 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f6508cd46f5d42c380834ba015c5e662 2024-11-11T03:40:17,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/51f626f1635f44949d115db3fcca6cd8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/51f626f1635f44949d115db3fcca6cd8 2024-11-11T03:40:17,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/41ff48bef1894c8ca696d77d25b52582 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/41ff48bef1894c8ca696d77d25b52582 2024-11-11T03:40:17,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/07355ab769f44ec78cce2b7757485ed4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/07355ab769f44ec78cce2b7757485ed4 2024-11-11T03:40:17,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/10c8466b898343cb9de2896611c5b5e2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/10c8466b898343cb9de2896611c5b5e2 2024-11-11T03:40:17,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/388b2a4fdcac48f3bdb90648b5f5f1c0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/388b2a4fdcac48f3bdb90648b5f5f1c0 2024-11-11T03:40:17,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/407c2bc62f8e4f9f880eed7e66e5f450 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/407c2bc62f8e4f9f880eed7e66e5f450 2024-11-11T03:40:17,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/9a735f880da8427a916073c11ffb24aa to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/9a735f880da8427a916073c11ffb24aa 2024-11-11T03:40:17,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f64846bf826f4daeb5d13335b3569a8b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/f64846bf826f4daeb5d13335b3569a8b 2024-11-11T03:40:17,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/615d642e511a42ea83e403182ae905c6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/615d642e511a42ea83e403182ae905c6 2024-11-11T03:40:17,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/3c54907b8f0b4bcfa1197dcadb5f60b8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/3c54907b8f0b4bcfa1197dcadb5f60b8 2024-11-11T03:40:17,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ce1a23298b114a25b12eeefcbd2b9c92 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ce1a23298b114a25b12eeefcbd2b9c92 2024-11-11T03:40:17,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/aacbb1263b764f8bb0f47f2d94a8f590 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/aacbb1263b764f8bb0f47f2d94a8f590 2024-11-11T03:40:17,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/313d14c4df1440eaa2792580a5750a65 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/313d14c4df1440eaa2792580a5750a65 2024-11-11T03:40:17,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/4cc5dfed9b9a48228163c7ddfc16b4c4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/4cc5dfed9b9a48228163c7ddfc16b4c4 2024-11-11T03:40:17,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/453acc9560d9448aa80a6f3309060af2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/453acc9560d9448aa80a6f3309060af2 2024-11-11T03:40:17,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/c63b47703d194e659940cfa04e0dff52 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/c63b47703d194e659940cfa04e0dff52 2024-11-11T03:40:17,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/dc70577c555f4b29a74aadaa15197ecf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/dc70577c555f4b29a74aadaa15197ecf 2024-11-11T03:40:17,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ccb78ab7b1e24af89f952fbd173568bb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/ccb78ab7b1e24af89f952fbd173568bb 2024-11-11T03:40:17,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e27f599fac6e47018b0c5c6cf3b7103f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/e27f599fac6e47018b0c5c6cf3b7103f 2024-11-11T03:40:17,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1369c0b92a443e5aedd91dc7b137549, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/643999685d444008aaf9560fc60409f0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c3a8aa6191d74a958bf8aaf26862b798, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/09cb51907aae4e89bec03fbb4ba5f26c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cec5dc9e752a48a784b826dd8d9f6182, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/62b1a93add964d83a1064a8bf294b922, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ad69aeb05b204b529c8dbedff7fe5a3f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c2e8f8d293004da5b161ecd423846bff, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d2e84a1cf8c4418291d207c2fafd8518, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f4980bd179ae418b90805b91c40fb0df, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cc494ba7e9eb481c8d3097b73dc441d7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eca5b6b317584e2bb58e5f4f9560351b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/49770f0689174ea4b421b4715280a561, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/8f8579ce33ec4caea80ef17fc7944dcd, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/3b4bf88947a142b6b650b802da534115, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/2bb417c5ee764f8e91711851c3d341da, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e30f74ea2ee14d95a200f8dbf43c78d7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bc3efe93a5634099b69bf903b818ab3d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d0e85fc02d414f4086f94d0e9775e02a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ecf50f58d18b4b7c86de9ec65f79ef62, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b07582b1ac7d47fdbe7e244832af9611, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/43c2f6a9e4d84570b9b2981c75c0f094, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/78c052439537498b881517d1f58de416, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e93e47b080af45778eb73b6c3fe54af5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/89b3916ade6d4dc3988b54aeae27c718, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/686ad96e6c5949fdb0abbcd4ec4f2be0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d04f5393605146739238197d6ed71fe2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ee387ef6fa8244289f152fa5d90b612f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/56d5be35c8f847a2988754b463af1081, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bfe34e042f5c4dada73579eb67805636, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/5a9a6302452f4872a9890f021ea2e16b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/728dd183a76742638ed7eaab1f76f4ae, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/dcb9064658824e6e9ce61bc2dc32ad0c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/deaf6e92ba2b4221bbb675aee8148544, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eed49c8ded2c4b48ab7ad59dc155d704] to archive 2024-11-11T03:40:17,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:40:17,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1369c0b92a443e5aedd91dc7b137549 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1369c0b92a443e5aedd91dc7b137549 2024-11-11T03:40:17,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/643999685d444008aaf9560fc60409f0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/643999685d444008aaf9560fc60409f0 2024-11-11T03:40:17,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c3a8aa6191d74a958bf8aaf26862b798 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c3a8aa6191d74a958bf8aaf26862b798 2024-11-11T03:40:17,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/09cb51907aae4e89bec03fbb4ba5f26c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/09cb51907aae4e89bec03fbb4ba5f26c 2024-11-11T03:40:17,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cec5dc9e752a48a784b826dd8d9f6182 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cec5dc9e752a48a784b826dd8d9f6182 2024-11-11T03:40:17,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/62b1a93add964d83a1064a8bf294b922 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/62b1a93add964d83a1064a8bf294b922 2024-11-11T03:40:17,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ad69aeb05b204b529c8dbedff7fe5a3f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ad69aeb05b204b529c8dbedff7fe5a3f 2024-11-11T03:40:17,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c2e8f8d293004da5b161ecd423846bff to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/c2e8f8d293004da5b161ecd423846bff 2024-11-11T03:40:17,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d2e84a1cf8c4418291d207c2fafd8518 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d2e84a1cf8c4418291d207c2fafd8518 2024-11-11T03:40:17,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f4980bd179ae418b90805b91c40fb0df to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f4980bd179ae418b90805b91c40fb0df 2024-11-11T03:40:17,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cc494ba7e9eb481c8d3097b73dc441d7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/cc494ba7e9eb481c8d3097b73dc441d7 2024-11-11T03:40:17,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eca5b6b317584e2bb58e5f4f9560351b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eca5b6b317584e2bb58e5f4f9560351b 2024-11-11T03:40:17,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/49770f0689174ea4b421b4715280a561 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/49770f0689174ea4b421b4715280a561 2024-11-11T03:40:17,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/8f8579ce33ec4caea80ef17fc7944dcd to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/8f8579ce33ec4caea80ef17fc7944dcd 2024-11-11T03:40:17,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/3b4bf88947a142b6b650b802da534115 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/3b4bf88947a142b6b650b802da534115 2024-11-11T03:40:17,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/2bb417c5ee764f8e91711851c3d341da to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/2bb417c5ee764f8e91711851c3d341da 2024-11-11T03:40:17,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e30f74ea2ee14d95a200f8dbf43c78d7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e30f74ea2ee14d95a200f8dbf43c78d7 2024-11-11T03:40:17,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bc3efe93a5634099b69bf903b818ab3d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bc3efe93a5634099b69bf903b818ab3d 2024-11-11T03:40:17,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d0e85fc02d414f4086f94d0e9775e02a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d0e85fc02d414f4086f94d0e9775e02a 2024-11-11T03:40:17,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ecf50f58d18b4b7c86de9ec65f79ef62 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ecf50f58d18b4b7c86de9ec65f79ef62 2024-11-11T03:40:17,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b07582b1ac7d47fdbe7e244832af9611 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b07582b1ac7d47fdbe7e244832af9611 2024-11-11T03:40:17,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/43c2f6a9e4d84570b9b2981c75c0f094 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/43c2f6a9e4d84570b9b2981c75c0f094 2024-11-11T03:40:17,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/78c052439537498b881517d1f58de416 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/78c052439537498b881517d1f58de416 2024-11-11T03:40:17,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e93e47b080af45778eb73b6c3fe54af5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/e93e47b080af45778eb73b6c3fe54af5 2024-11-11T03:40:17,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/89b3916ade6d4dc3988b54aeae27c718 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/89b3916ade6d4dc3988b54aeae27c718 2024-11-11T03:40:17,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/686ad96e6c5949fdb0abbcd4ec4f2be0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/686ad96e6c5949fdb0abbcd4ec4f2be0 2024-11-11T03:40:17,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d04f5393605146739238197d6ed71fe2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/d04f5393605146739238197d6ed71fe2 2024-11-11T03:40:17,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ee387ef6fa8244289f152fa5d90b612f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ee387ef6fa8244289f152fa5d90b612f 2024-11-11T03:40:17,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/56d5be35c8f847a2988754b463af1081 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/56d5be35c8f847a2988754b463af1081 2024-11-11T03:40:17,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bfe34e042f5c4dada73579eb67805636 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/bfe34e042f5c4dada73579eb67805636 2024-11-11T03:40:17,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/5a9a6302452f4872a9890f021ea2e16b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/5a9a6302452f4872a9890f021ea2e16b 2024-11-11T03:40:17,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/728dd183a76742638ed7eaab1f76f4ae to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/728dd183a76742638ed7eaab1f76f4ae 2024-11-11T03:40:17,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/dcb9064658824e6e9ce61bc2dc32ad0c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/dcb9064658824e6e9ce61bc2dc32ad0c 2024-11-11T03:40:17,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/deaf6e92ba2b4221bbb675aee8148544 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/deaf6e92ba2b4221bbb675aee8148544 2024-11-11T03:40:17,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eed49c8ded2c4b48ab7ad59dc155d704 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/eed49c8ded2c4b48ab7ad59dc155d704 2024-11-11T03:40:17,249 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=1 2024-11-11T03:40:17,250 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. 2024-11-11T03:40:17,250 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 488bb5013d196b74e2d5128b2d5b7358: Waiting for close lock at 1731296414758Running coprocessor pre-close hooks at 1731296414758Disabling compacts and flushes for region at 1731296414758Disabling writes for close at 1731296415816 (+1058 ms)Obtaining lock to block concurrent updates at 1731296415816Preparing flush snapshotting stores in 488bb5013d196b74e2d5128b2d5b7358 at 1731296415816Finished memstore snapshotting TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731296415817 (+1 ms)Flushing stores of TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358. at 1731296415817Flushing 488bb5013d196b74e2d5128b2d5b7358/A: creating writer at 1731296415817Flushing 488bb5013d196b74e2d5128b2d5b7358/A: appending metadata at 1731296415820 (+3 ms)Flushing 488bb5013d196b74e2d5128b2d5b7358/A: closing flushed file at 1731296415820Flushing 488bb5013d196b74e2d5128b2d5b7358/B: creating writer at 1731296416230 (+410 ms)Flushing 488bb5013d196b74e2d5128b2d5b7358/B: appending metadata at 1731296416233 (+3 ms)Flushing 488bb5013d196b74e2d5128b2d5b7358/B: closing flushed file at 1731296416233Flushing 488bb5013d196b74e2d5128b2d5b7358/C: creating writer at 1731296416651 (+418 ms)Flushing 488bb5013d196b74e2d5128b2d5b7358/C: appending metadata at 1731296416654 (+3 ms)Flushing 488bb5013d196b74e2d5128b2d5b7358/C: closing flushed file at 1731296416655 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77598a39: reopening flushed file at 1731296417065 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15aae9e3: reopening flushed file at 1731296417071 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fd40987: reopening flushed file at 1731296417076 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 488bb5013d196b74e2d5128b2d5b7358 in 1266ms, sequenceid=544, compaction requested=true at 1731296417082 (+6 ms)Writing region close event to WAL at 1731296417246 (+164 ms)Running coprocessor post-close hooks at 1731296417250 (+4 ms)Closed at 1731296417250 2024-11-11T03:40:17,252 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:17,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=488bb5013d196b74e2d5128b2d5b7358, regionState=CLOSED 2024-11-11T03:40:17,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:40:17,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T03:40:17,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 488bb5013d196b74e2d5128b2d5b7358, server=3ffe7f9d089e,34473,1731296384288 in 2.6550 sec 2024-11-11T03:40:17,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-11T03:40:17,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=488bb5013d196b74e2d5128b2d5b7358, UNASSIGN in 2.6620 sec 2024-11-11T03:40:17,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T03:40:17,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6680 sec 2024-11-11T03:40:17,262 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296417262"}]},"ts":"1731296417262"} 2024-11-11T03:40:17,263 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T03:40:17,263 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T03:40:17,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6830 sec 2024-11-11T03:40:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T03:40:18,744 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:18,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T03:40:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:18,758 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T03:40:18,760 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:18,764 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:18,768 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/recovered.edits] 2024-11-11T03:40:18,771 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/158067a3d8fa481f929f83aa34adb8f9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/158067a3d8fa481f929f83aa34adb8f9 2024-11-11T03:40:18,773 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/263091133cea492c82fb926cbe035560 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/263091133cea492c82fb926cbe035560 2024-11-11T03:40:18,774 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8e3dafc324bd43158d948fa0fddc8fe6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/A/8e3dafc324bd43158d948fa0fddc8fe6 2024-11-11T03:40:18,776 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2ea61679d083489e847924e031d0a3f5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/2ea61679d083489e847924e031d0a3f5 2024-11-11T03:40:18,778 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a4a00b546ebc4103b37842977ed658a5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/a4a00b546ebc4103b37842977ed658a5 2024-11-11T03:40:18,779 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/df6483beafd14e3e882d7ef0f1689c15 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/B/df6483beafd14e3e882d7ef0f1689c15 2024-11-11T03:40:18,781 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ab722af8eff54810820033e08c9cf206 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/ab722af8eff54810820033e08c9cf206 2024-11-11T03:40:18,782 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b46ed14e2fa749cdb2e66247ede6df77 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/b46ed14e2fa749cdb2e66247ede6df77 2024-11-11T03:40:18,783 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1ff77fdfea04111b6dc244a828ee02d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/C/f1ff77fdfea04111b6dc244a828ee02d 2024-11-11T03:40:18,786 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/recovered.edits/547.seqid to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358/recovered.edits/547.seqid 2024-11-11T03:40:18,786 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/488bb5013d196b74e2d5128b2d5b7358 2024-11-11T03:40:18,787 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T03:40:18,792 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:18,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T03:40:18,802 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T03:40:18,829 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T03:40:18,830 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:18,830 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T03:40:18,831 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731296418830"}]},"ts":"9223372036854775807"} 2024-11-11T03:40:18,835 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T03:40:18,835 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 488bb5013d196b74e2d5128b2d5b7358, NAME => 'TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T03:40:18,835 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T03:40:18,835 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731296418835"}]},"ts":"9223372036854775807"} 2024-11-11T03:40:18,838 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T03:40:18,839 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:18,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-11T03:40:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T03:40:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T03:40:18,862 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:18,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS:0;3ffe7f9d089e:34473-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1022135714_22 at /127.0.0.1:56198 [Waiting for operation #201] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383580093_22 at /127.0.0.1:60448 [Waiting for operation #256] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 147) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12089 (was 12892) 2024-11-11T03:40:18,881 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=12088 2024-11-11T03:40:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:40:18,883 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:40:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:18,885 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T03:40:18,885 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:18,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-11T03:40:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T03:40:18,887 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T03:40:18,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741953_1129 (size=1061) 2024-11-11T03:40:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T03:40:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T03:40:19,299 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:40:19,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741954_1130 (size=53) 2024-11-11T03:40:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T03:40:19,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:40:19,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 887247b757faef0e5f5ee791239fb929, disabling compactions & flushes 2024-11-11T03:40:19,711 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:19,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:19,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. after waiting 0 ms 2024-11-11T03:40:19,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:19,712 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:19,712 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 887247b757faef0e5f5ee791239fb929: Waiting for close lock at 1731296419711Disabling compacts and flushes for region at 1731296419711Disabling writes for close at 1731296419712 (+1 ms)Writing region close event to WAL at 1731296419712Closed at 1731296419712 2024-11-11T03:40:19,715 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T03:40:19,716 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731296419715"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731296419715"}]},"ts":"1731296419715"} 2024-11-11T03:40:19,718 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T03:40:19,719 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T03:40:19,719 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296419719"}]},"ts":"1731296419719"} 2024-11-11T03:40:19,721 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T03:40:19,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, ASSIGN}] 2024-11-11T03:40:19,722 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, ASSIGN 2024-11-11T03:40:19,723 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, ASSIGN; state=OFFLINE, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=false 2024-11-11T03:40:19,875 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:19,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, ASSIGN because future has completed 2024-11-11T03:40:19,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:40:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T03:40:20,043 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:20,043 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:40:20,043 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,043 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:40:20,044 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,044 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,046 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,048 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:20,049 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 887247b757faef0e5f5ee791239fb929 columnFamilyName A 2024-11-11T03:40:20,049 DEBUG [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:20,050 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(327): Store=887247b757faef0e5f5ee791239fb929/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:20,050 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,051 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:20,051 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 887247b757faef0e5f5ee791239fb929 columnFamilyName B 2024-11-11T03:40:20,051 DEBUG [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:20,052 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(327): Store=887247b757faef0e5f5ee791239fb929/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:20,052 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,053 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:20,053 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 887247b757faef0e5f5ee791239fb929 columnFamilyName C 2024-11-11T03:40:20,053 DEBUG [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:20,054 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(327): Store=887247b757faef0e5f5ee791239fb929/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:20,054 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:20,054 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,054 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,055 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,056 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,056 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,056 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:40:20,057 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,059 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:40:20,059 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 887247b757faef0e5f5ee791239fb929; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69291385, jitterRate=0.03252209722995758}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:40:20,059 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:20,060 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 887247b757faef0e5f5ee791239fb929: Running coprocessor pre-open hook at 1731296420044Writing region info on filesystem at 1731296420044Initializing all the Stores at 1731296420045 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296420045Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296420045Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296420045Cleaning up temporary data from old regions at 1731296420056 (+11 ms)Running coprocessor post-open hooks at 1731296420059 (+3 ms)Region opened successfully at 1731296420060 (+1 ms) 2024-11-11T03:40:20,060 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., pid=32, masterSystemTime=1731296420034 2024-11-11T03:40:20,062 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:20,062 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:20,063 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=OPEN, openSeqNum=2, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:20,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:40:20,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T03:40:20,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 in 184 msec 2024-11-11T03:40:20,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-11T03:40:20,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, ASSIGN in 346 msec 2024-11-11T03:40:20,069 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T03:40:20,069 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296420069"}]},"ts":"1731296420069"} 2024-11-11T03:40:20,071 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T03:40:20,071 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T03:40:20,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-11-11T03:40:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T03:40:21,044 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:21,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:21,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:21,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:21,051 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37afb74d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:21,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236dd13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:21,056 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:21,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:21,060 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T03:40:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ffe7f9d089e,45295,1731296383499 2024-11-11T03:40:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44cff70e 2024-11-11T03:40:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:40:21,062 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:40:21,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:40:21,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:40:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:21,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741955_1131 (size=1097) 2024-11-11T03:40:21,490 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T03:40:21,491 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T03:40:21,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:40:21,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, REOPEN/MOVE}] 2024-11-11T03:40:21,500 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, REOPEN/MOVE 2024-11-11T03:40:21,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:21,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, REOPEN/MOVE because future has completed 2024-11-11T03:40:21,503 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:40:21,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:40:21,656 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:21,656 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:40:21,656 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 887247b757faef0e5f5ee791239fb929, disabling compactions & flushes 2024-11-11T03:40:21,656 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:21,656 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:21,656 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. after waiting 0 ms 2024-11-11T03:40:21,656 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:21,661 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T03:40:21,662 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:21,662 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 887247b757faef0e5f5ee791239fb929: Waiting for close lock at 1731296421656Running coprocessor pre-close hooks at 1731296421656Disabling compacts and flushes for region at 1731296421656Disabling writes for close at 1731296421656Writing region close event to WAL at 1731296421657 (+1 ms)Running coprocessor post-close hooks at 1731296421662 (+5 ms)Closed at 1731296421662 2024-11-11T03:40:21,662 WARN [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 887247b757faef0e5f5ee791239fb929 to self. 2024-11-11T03:40:21,664 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:21,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=CLOSED 2024-11-11T03:40:21,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:40:21,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T03:40:21,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 in 166 msec 2024-11-11T03:40:21,672 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, REOPEN/MOVE; state=CLOSED, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=true 2024-11-11T03:40:21,823 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:21,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, REOPEN/MOVE because future has completed 2024-11-11T03:40:21,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:40:21,991 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:21,992 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:40:21,992 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:21,992 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:40:21,992 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:21,992 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:21,998 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:21,999 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:22,005 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 887247b757faef0e5f5ee791239fb929 columnFamilyName A 2024-11-11T03:40:22,007 DEBUG [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:22,007 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(327): Store=887247b757faef0e5f5ee791239fb929/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:22,007 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,008 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:22,008 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 887247b757faef0e5f5ee791239fb929 columnFamilyName B 2024-11-11T03:40:22,008 DEBUG [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:22,008 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(327): Store=887247b757faef0e5f5ee791239fb929/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:22,009 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,009 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:22,009 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 887247b757faef0e5f5ee791239fb929 columnFamilyName C 2024-11-11T03:40:22,009 DEBUG [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:22,010 INFO [StoreOpener-887247b757faef0e5f5ee791239fb929-1 {}] regionserver.HStore(327): Store=887247b757faef0e5f5ee791239fb929/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:22,010 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,010 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,010 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,011 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,012 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,012 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,013 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:40:22,014 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,015 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 887247b757faef0e5f5ee791239fb929; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65941650, jitterRate=-0.017392843961715698}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:40:22,015 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,017 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 887247b757faef0e5f5ee791239fb929: Running coprocessor pre-open hook at 1731296421992Writing region info on filesystem at 1731296421992Initializing all the Stores at 1731296421994 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296421994Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296421994Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296421994Cleaning up temporary data from old regions at 1731296422012 (+18 ms)Running coprocessor post-open hooks at 1731296422015 (+3 ms)Region opened successfully at 1731296422017 (+2 ms) 2024-11-11T03:40:22,018 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., pid=37, masterSystemTime=1731296421983 2024-11-11T03:40:22,020 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,020 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,021 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=OPEN, openSeqNum=5, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:40:22,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-11T03:40:22,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 in 195 msec 2024-11-11T03:40:22,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T03:40:22,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, REOPEN/MOVE in 526 msec 2024-11-11T03:40:22,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T03:40:22,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-11T03:40:22,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-11T03:40:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T03:40:22,037 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:22,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8adca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,041 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffb4dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,043 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54499fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,045 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9dd9d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,047 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5616f047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,049 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc5c21e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,051 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1638f527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39aa9a1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,055 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a69dae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,057 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692aecfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,059 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1914d987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,061 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56428ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,063 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8238c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12162879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,067 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1526760f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,069 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2570f4e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,070 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:22,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:22,071 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5352d308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:22,073 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:22,074 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,074 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,074 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,074 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,074 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:22,075 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,076 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-11T03:40:22,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,079 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,079 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T03:40:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:22,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:22,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:40:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:40:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:22,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d898a7341ece45a4a74cad117077a609_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296422094/Put/seqid=0 2024-11-11T03:40:22,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296422140, max timestamp=1731296422151Num uniques 859; 2024-11-11T03:40:22,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296422140, max timestamp=1731296422152Num uniques 859; 2024-11-11T03:40:22,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296422140, max timestamp=1731296422152Num uniques 859; 2024-11-11T03:40:22,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:22,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:22,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:22,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741956_1132 (size=17034) 2024-11-11T03:40:22,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:22,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d898a7341ece45a4a74cad117077a609_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d898a7341ece45a4a74cad117077a609_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:22,190 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/d3609efe2eaa42409522e84ef5787d28, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T03:40:22,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/d3609efe2eaa42409522e84ef5787d28 is 175, key is test_row_0/A:col10/1731296422094/Put/seqid=0 2024-11-11T03:40:22,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741957_1133 (size=48139) 2024-11-11T03:40:22,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/d3609efe2eaa42409522e84ef5787d28 2024-11-11T03:40:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296482193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296482198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296482196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296482201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296482206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T03:40:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:22,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4f625d685b5b4e0eb109bc3b04a5e77b is 50, key is test_row_0/B:col10/1731296422094/Put/seqid=0 2024-11-11T03:40:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741958_1134 (size=12001) 2024-11-11T03:40:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296482322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296482323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296482323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296482323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296482323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T03:40:22,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:22,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T03:40:22,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296482532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296482532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296482532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296482542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296482542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T03:40:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4f625d685b5b4e0eb109bc3b04a5e77b 2024-11-11T03:40:22,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/2a81ccbf5e894e26b6e4055532c58688 is 50, key is test_row_0/C:col10/1731296422094/Put/seqid=0 2024-11-11T03:40:22,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T03:40:22,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:22,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741959_1135 (size=12001) 2024-11-11T03:40:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T03:40:22,806 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:40:22,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296482843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296482843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296482843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T03:40:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:22,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296482852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296482852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T03:40:23,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:23,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:23,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:23,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:23,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:23,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/2a81ccbf5e894e26b6e4055532c58688 2024-11-11T03:40:23,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/d3609efe2eaa42409522e84ef5787d28 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/d3609efe2eaa42409522e84ef5787d28 2024-11-11T03:40:23,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/d3609efe2eaa42409522e84ef5787d28, entries=250, sequenceid=14, filesize=47.0 K 2024-11-11T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4f625d685b5b4e0eb109bc3b04a5e77b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4f625d685b5b4e0eb109bc3b04a5e77b 2024-11-11T03:40:23,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4f625d685b5b4e0eb109bc3b04a5e77b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T03:40:23,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/2a81ccbf5e894e26b6e4055532c58688 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/2a81ccbf5e894e26b6e4055532c58688 2024-11-11T03:40:23,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/2a81ccbf5e894e26b6e4055532c58688, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T03:40:23,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 887247b757faef0e5f5ee791239fb929 in 1040ms, sequenceid=14, compaction requested=false 2024-11-11T03:40:23,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:23,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T03:40:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:23,163 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T03:40:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154cb02f029cd4d7d91626e29a342d502_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296422195/Put/seqid=0 2024-11-11T03:40:23,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741960_1136 (size=12154) 2024-11-11T03:40:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:23,204 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154cb02f029cd4d7d91626e29a342d502_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154cb02f029cd4d7d91626e29a342d502_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55130c3d75d484b97bf429f4d76f0fb, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55130c3d75d484b97bf429f4d76f0fb is 175, key is test_row_0/A:col10/1731296422195/Put/seqid=0 2024-11-11T03:40:23,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741961_1137 (size=30955) 2024-11-11T03:40:23,220 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55130c3d75d484b97bf429f4d76f0fb 2024-11-11T03:40:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T03:40:23,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3d8c5b299ec649deba1c1876991cb2fd is 50, key is test_row_0/B:col10/1731296422195/Put/seqid=0 2024-11-11T03:40:23,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741962_1138 (size=12001) 2024-11-11T03:40:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:23,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296483367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296483367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296483369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296483370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296483372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296483483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296483483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296483483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296483483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,653 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3d8c5b299ec649deba1c1876991cb2fd 2024-11-11T03:40:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/653eee7d9f7747cfb892f4a85993e94d is 50, key is test_row_0/C:col10/1731296422195/Put/seqid=0 2024-11-11T03:40:23,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741963_1139 (size=12001) 2024-11-11T03:40:23,673 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/653eee7d9f7747cfb892f4a85993e94d 2024-11-11T03:40:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55130c3d75d484b97bf429f4d76f0fb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55130c3d75d484b97bf429f4d76f0fb 2024-11-11T03:40:23,690 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55130c3d75d484b97bf429f4d76f0fb, entries=150, sequenceid=50, filesize=30.2 K 2024-11-11T03:40:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3d8c5b299ec649deba1c1876991cb2fd as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3d8c5b299ec649deba1c1876991cb2fd 2024-11-11T03:40:23,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296483702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296483702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296483702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296483703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:23,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3d8c5b299ec649deba1c1876991cb2fd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T03:40:23,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/653eee7d9f7747cfb892f4a85993e94d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/653eee7d9f7747cfb892f4a85993e94d 2024-11-11T03:40:23,717 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/653eee7d9f7747cfb892f4a85993e94d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T03:40:23,718 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 887247b757faef0e5f5ee791239fb929 in 555ms, sequenceid=50, compaction requested=false 2024-11-11T03:40:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-11T03:40:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-11T03:40:23,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T03:40:23,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-11-11T03:40:23,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6490 sec 2024-11-11T03:40:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:24,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T03:40:24,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:24,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111156b776cd2a204cdcba53d0a1dd2bf7e3_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296424014/Put/seqid=0 2024-11-11T03:40:24,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741964_1140 (size=12154) 2024-11-11T03:40:24,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296484097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296484095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296484105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296484106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296484212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296484212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296484222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296484222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T03:40:24,232 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:24,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T03:40:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T03:40:24,236 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:24,237 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:24,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T03:40:24,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T03:40:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:24,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296484391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,393 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296484432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296484432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296484432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296484432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,460 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:24,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111156b776cd2a204cdcba53d0a1dd2bf7e3_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156b776cd2a204cdcba53d0a1dd2bf7e3_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:24,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0f1525138914495a821173d29f676273, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:24,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0f1525138914495a821173d29f676273 is 175, key is test_row_0/A:col10/1731296424014/Put/seqid=0 2024-11-11T03:40:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741965_1141 (size=30955) 2024-11-11T03:40:24,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0f1525138914495a821173d29f676273 2024-11-11T03:40:24,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/f4f75e51b217476e839be38bec0501e7 is 50, key is test_row_0/B:col10/1731296424014/Put/seqid=0 2024-11-11T03:40:24,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741966_1142 (size=12001) 2024-11-11T03:40:24,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/f4f75e51b217476e839be38bec0501e7 2024-11-11T03:40:24,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/6e126d23c35b47d885b9dc8760d643c8 is 50, key is test_row_0/C:col10/1731296424014/Put/seqid=0 2024-11-11T03:40:24,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741967_1143 (size=12001) 2024-11-11T03:40:24,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/6e126d23c35b47d885b9dc8760d643c8 2024-11-11T03:40:24,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T03:40:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T03:40:24,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0f1525138914495a821173d29f676273 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0f1525138914495a821173d29f676273 2024-11-11T03:40:24,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0f1525138914495a821173d29f676273, entries=150, sequenceid=63, filesize=30.2 K 2024-11-11T03:40:24,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/f4f75e51b217476e839be38bec0501e7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/f4f75e51b217476e839be38bec0501e7 2024-11-11T03:40:24,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/f4f75e51b217476e839be38bec0501e7, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T03:40:24,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/6e126d23c35b47d885b9dc8760d643c8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6e126d23c35b47d885b9dc8760d643c8 2024-11-11T03:40:24,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6e126d23c35b47d885b9dc8760d643c8, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T03:40:24,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 887247b757faef0e5f5ee791239fb929 in 573ms, sequenceid=63, compaction requested=true 2024-11-11T03:40:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:24,588 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:24,588 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:24,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:24,589 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:24,589 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:24,589 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:24,589 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:24,590 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,590 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,590 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4f625d685b5b4e0eb109bc3b04a5e77b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3d8c5b299ec649deba1c1876991cb2fd, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/f4f75e51b217476e839be38bec0501e7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.2 K 2024-11-11T03:40:24,590 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/d3609efe2eaa42409522e84ef5787d28, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55130c3d75d484b97bf429f4d76f0fb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0f1525138914495a821173d29f676273] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=107.5 K 2024-11-11T03:40:24,590 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,590 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f625d685b5b4e0eb109bc3b04a5e77b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731296422084 2024-11-11T03:40:24,590 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/d3609efe2eaa42409522e84ef5787d28, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55130c3d75d484b97bf429f4d76f0fb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0f1525138914495a821173d29f676273] 2024-11-11T03:40:24,591 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d8c5b299ec649deba1c1876991cb2fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731296422195 2024-11-11T03:40:24,591 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3609efe2eaa42409522e84ef5787d28, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731296422084 2024-11-11T03:40:24,591 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55130c3d75d484b97bf429f4d76f0fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731296422195 2024-11-11T03:40:24,591 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1525138914495a821173d29f676273, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731296423366 2024-11-11T03:40:24,592 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f75e51b217476e839be38bec0501e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731296423366 2024-11-11T03:40:24,600 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:24,602 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:24,603 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/557e28fa7d3448fdaacbe1f462f4ca0b is 50, key is test_row_0/B:col10/1731296424014/Put/seqid=0 2024-11-11T03:40:24,607 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111546a0995964943d7b6ccc870a868070d_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:24,612 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111546a0995964943d7b6ccc870a868070d_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:24,612 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111546a0995964943d7b6ccc870a868070d_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:24,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741968_1144 (size=12104) 2024-11-11T03:40:24,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741969_1145 (size=4469) 2024-11-11T03:40:24,637 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#126 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:24,638 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/cd12e64ff6e54cfa90c4cfaed2489350 is 175, key is test_row_0/A:col10/1731296424014/Put/seqid=0 2024-11-11T03:40:24,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741970_1146 (size=31058) 2024-11-11T03:40:24,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T03:40:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:24,698 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T03:40:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116dadfd02d6a84b04b21858c8f479a8eb_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296424091/Put/seqid=0 2024-11-11T03:40:24,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741971_1147 (size=12154) 2024-11-11T03:40:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:24,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296484755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296484756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296484757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296484758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T03:40:24,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296484872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296484872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296484873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296484873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,033 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/557e28fa7d3448fdaacbe1f462f4ca0b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/557e28fa7d3448fdaacbe1f462f4ca0b 2024-11-11T03:40:25,042 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into 557e28fa7d3448fdaacbe1f462f4ca0b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:25,043 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:25,043 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=13, startTime=1731296424588; duration=0sec 2024-11-11T03:40:25,043 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:25,043 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:25,043 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:25,044 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:25,044 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:25,045 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:25,045 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/2a81ccbf5e894e26b6e4055532c58688, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/653eee7d9f7747cfb892f4a85993e94d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6e126d23c35b47d885b9dc8760d643c8] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.2 K 2024-11-11T03:40:25,045 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a81ccbf5e894e26b6e4055532c58688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731296422084 2024-11-11T03:40:25,046 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 653eee7d9f7747cfb892f4a85993e94d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731296422195 2024-11-11T03:40:25,046 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e126d23c35b47d885b9dc8760d643c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731296423366 2024-11-11T03:40:25,056 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:25,057 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/553ea3f2270942959e78194b90559c8f is 50, key is test_row_0/C:col10/1731296424014/Put/seqid=0 2024-11-11T03:40:25,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741972_1148 (size=12104) 2024-11-11T03:40:25,071 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/cd12e64ff6e54cfa90c4cfaed2489350 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/cd12e64ff6e54cfa90c4cfaed2489350 2024-11-11T03:40:25,078 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into cd12e64ff6e54cfa90c4cfaed2489350(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:25,078 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:25,078 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=13, startTime=1731296424588; duration=0sec 2024-11-11T03:40:25,078 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:25,078 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296485082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296485082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296485092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296485092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:25,126 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116dadfd02d6a84b04b21858c8f479a8eb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116dadfd02d6a84b04b21858c8f479a8eb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:25,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/b5c41cc0c1ff4c2cbee938fc265cf21b, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/b5c41cc0c1ff4c2cbee938fc265cf21b is 175, key is test_row_0/A:col10/1731296424091/Put/seqid=0 2024-11-11T03:40:25,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741973_1149 (size=30955) 2024-11-11T03:40:25,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/b5c41cc0c1ff4c2cbee938fc265cf21b 2024-11-11T03:40:25,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/cb1fbf8a88194bfca272747afe690ec2 is 50, key is test_row_0/B:col10/1731296424091/Put/seqid=0 2024-11-11T03:40:25,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741974_1150 (size=12001) 2024-11-11T03:40:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T03:40:25,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296485392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296485392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296485402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296485402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,470 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/553ea3f2270942959e78194b90559c8f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/553ea3f2270942959e78194b90559c8f 2024-11-11T03:40:25,476 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into 553ea3f2270942959e78194b90559c8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:25,476 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:25,476 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=13, startTime=1731296424588; duration=0sec 2024-11-11T03:40:25,476 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:25,476 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:25,554 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/cb1fbf8a88194bfca272747afe690ec2 2024-11-11T03:40:25,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/f7625d34419f4670a86eaf482f99a7f9 is 50, key is test_row_0/C:col10/1731296424091/Put/seqid=0 2024-11-11T03:40:25,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741975_1151 (size=12001) 2024-11-11T03:40:25,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296485902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296485902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296485912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296485912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:25,967 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/f7625d34419f4670a86eaf482f99a7f9 2024-11-11T03:40:25,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/b5c41cc0c1ff4c2cbee938fc265cf21b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/b5c41cc0c1ff4c2cbee938fc265cf21b 2024-11-11T03:40:25,979 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/b5c41cc0c1ff4c2cbee938fc265cf21b, entries=150, sequenceid=82, filesize=30.2 K 2024-11-11T03:40:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/cb1fbf8a88194bfca272747afe690ec2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/cb1fbf8a88194bfca272747afe690ec2 2024-11-11T03:40:25,991 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/cb1fbf8a88194bfca272747afe690ec2, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T03:40:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/f7625d34419f4670a86eaf482f99a7f9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/f7625d34419f4670a86eaf482f99a7f9 2024-11-11T03:40:25,998 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/f7625d34419f4670a86eaf482f99a7f9, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T03:40:26,003 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 887247b757faef0e5f5ee791239fb929 in 1305ms, sequenceid=82, compaction requested=false 2024-11-11T03:40:26,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:26,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T03:40:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T03:40:26,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T03:40:26,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7690 sec 2024-11-11T03:40:26,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7770 sec 2024-11-11T03:40:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T03:40:26,382 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:26,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T03:40:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T03:40:26,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:26,386 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:26,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:26,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T03:40:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:26,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b901c4e4b1e4874ad9d15df26d277d9_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296424747/Put/seqid=0 2024-11-11T03:40:26,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741976_1152 (size=12154) 2024-11-11T03:40:26,449 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:26,460 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b901c4e4b1e4874ad9d15df26d277d9_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b901c4e4b1e4874ad9d15df26d277d9_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:26,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/382a889e8c91480ab4fca7f07b850cb0, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:26,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/382a889e8c91480ab4fca7f07b850cb0 is 175, key is test_row_0/A:col10/1731296424747/Put/seqid=0 2024-11-11T03:40:26,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741977_1153 (size=30955) 2024-11-11T03:40:26,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/382a889e8c91480ab4fca7f07b850cb0 2024-11-11T03:40:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T03:40:26,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d535b2f180bb4f8db4a91841e76bbe08 is 50, key is test_row_0/B:col10/1731296424747/Put/seqid=0 2024-11-11T03:40:26,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741978_1154 (size=12001) 2024-11-11T03:40:26,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d535b2f180bb4f8db4a91841e76bbe08 2024-11-11T03:40:26,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/8c0339df2f6b4799a1bdb36338b749c4 is 50, key is test_row_0/C:col10/1731296424747/Put/seqid=0 2024-11-11T03:40:26,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T03:40:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296486548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:26,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741979_1155 (size=12001) 2024-11-11T03:40:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296486661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:26,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T03:40:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T03:40:26,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T03:40:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:26,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296486872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:26,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296486912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:26,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296486912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:26,915 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:26,916 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296486922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296486922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:26,924 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,925 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:26,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/8c0339df2f6b4799a1bdb36338b749c4 2024-11-11T03:40:26,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/382a889e8c91480ab4fca7f07b850cb0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/382a889e8c91480ab4fca7f07b850cb0 2024-11-11T03:40:26,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/382a889e8c91480ab4fca7f07b850cb0, entries=150, sequenceid=99, filesize=30.2 K 2024-11-11T03:40:26,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d535b2f180bb4f8db4a91841e76bbe08 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d535b2f180bb4f8db4a91841e76bbe08 2024-11-11T03:40:26,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d535b2f180bb4f8db4a91841e76bbe08, entries=150, sequenceid=99, filesize=11.7 K 2024-11-11T03:40:26,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/8c0339df2f6b4799a1bdb36338b749c4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8c0339df2f6b4799a1bdb36338b749c4 2024-11-11T03:40:26,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8c0339df2f6b4799a1bdb36338b749c4, entries=150, sequenceid=99, filesize=11.7 K 2024-11-11T03:40:26,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 887247b757faef0e5f5ee791239fb929 in 562ms, sequenceid=99, compaction requested=true 2024-11-11T03:40:26,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:26,989 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:26,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:26,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:26,989 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:26,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:26,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:26,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:26,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:26,990 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:26,990 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:26,990 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,990 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/cd12e64ff6e54cfa90c4cfaed2489350, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/b5c41cc0c1ff4c2cbee938fc265cf21b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/382a889e8c91480ab4fca7f07b850cb0] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=90.8 K 2024-11-11T03:40:26,990 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,990 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/cd12e64ff6e54cfa90c4cfaed2489350, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/b5c41cc0c1ff4c2cbee938fc265cf21b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/382a889e8c91480ab4fca7f07b850cb0] 2024-11-11T03:40:26,991 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:26,991 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:26,991 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:26,991 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/557e28fa7d3448fdaacbe1f462f4ca0b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/cb1fbf8a88194bfca272747afe690ec2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d535b2f180bb4f8db4a91841e76bbe08] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.3 K 2024-11-11T03:40:26,991 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd12e64ff6e54cfa90c4cfaed2489350, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731296423366 2024-11-11T03:40:26,991 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 557e28fa7d3448fdaacbe1f462f4ca0b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731296423366 2024-11-11T03:40:26,991 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5c41cc0c1ff4c2cbee938fc265cf21b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731296424091 2024-11-11T03:40:26,992 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 382a889e8c91480ab4fca7f07b850cb0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731296424747 2024-11-11T03:40:26,992 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1fbf8a88194bfca272747afe690ec2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731296424091 2024-11-11T03:40:26,993 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d535b2f180bb4f8db4a91841e76bbe08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731296424747 2024-11-11T03:40:27,001 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:27,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T03:40:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:27,003 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T03:40:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:27,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3be73bcffe53489882171a10e741a8cc is 50, key is test_row_0/B:col10/1731296424747/Put/seqid=0 2024-11-11T03:40:27,006 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T03:40:27,027 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117d23de06d035417494729afc2afa71d4_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:27,030 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117d23de06d035417494729afc2afa71d4_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:27,030 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117d23de06d035417494729afc2afa71d4_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119e03258cd091474bbca66f0493f4d186_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296426513/Put/seqid=0 2024-11-11T03:40:27,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741980_1156 (size=12207) 2024-11-11T03:40:27,060 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3be73bcffe53489882171a10e741a8cc as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3be73bcffe53489882171a10e741a8cc 2024-11-11T03:40:27,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741981_1157 (size=4469) 2024-11-11T03:40:27,066 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#135 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:27,067 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/dd86286032824d098431d0c2f021d31b is 175, key is test_row_0/A:col10/1731296424747/Put/seqid=0 2024-11-11T03:40:27,068 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into 3be73bcffe53489882171a10e741a8cc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:27,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:27,069 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=13, startTime=1731296426989; duration=0sec 2024-11-11T03:40:27,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:27,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:27,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:27,070 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:27,070 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:27,070 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:27,070 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/553ea3f2270942959e78194b90559c8f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/f7625d34419f4670a86eaf482f99a7f9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8c0339df2f6b4799a1bdb36338b749c4] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.3 K 2024-11-11T03:40:27,070 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 553ea3f2270942959e78194b90559c8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731296423366 2024-11-11T03:40:27,071 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f7625d34419f4670a86eaf482f99a7f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731296424091 2024-11-11T03:40:27,071 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0339df2f6b4799a1bdb36338b749c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731296424747 2024-11-11T03:40:27,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741982_1158 (size=12154) 2024-11-11T03:40:27,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:27,080 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119e03258cd091474bbca66f0493f4d186_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119e03258cd091474bbca66f0493f4d186_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:27,082 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/8464e4493f774d6d91b01436110bd54d, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:27,082 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/a17a7de85fe64585801da8604f385064 is 50, key is test_row_0/C:col10/1731296424747/Put/seqid=0 2024-11-11T03:40:27,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/8464e4493f774d6d91b01436110bd54d is 175, key is test_row_0/A:col10/1731296426513/Put/seqid=0 2024-11-11T03:40:27,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741983_1159 (size=31161) 2024-11-11T03:40:27,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741984_1160 (size=30955) 2024-11-11T03:40:27,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741985_1161 (size=12207) 2024-11-11T03:40:27,110 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/a17a7de85fe64585801da8604f385064 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/a17a7de85fe64585801da8604f385064 2024-11-11T03:40:27,116 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into a17a7de85fe64585801da8604f385064(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:27,116 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:27,116 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=13, startTime=1731296426990; duration=0sec 2024-11-11T03:40:27,116 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:27,116 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:27,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296487225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:27,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296487332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:27,500 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/dd86286032824d098431d0c2f021d31b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/dd86286032824d098431d0c2f021d31b 2024-11-11T03:40:27,502 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/8464e4493f774d6d91b01436110bd54d 2024-11-11T03:40:27,510 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into dd86286032824d098431d0c2f021d31b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:27,510 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:27,510 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=13, startTime=1731296426989; duration=0sec 2024-11-11T03:40:27,510 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:27,510 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d52af7b428a94e31b40c0cca5bf6bd64 is 50, key is test_row_0/B:col10/1731296426513/Put/seqid=0 2024-11-11T03:40:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T03:40:27,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741986_1162 (size=12001) 2024-11-11T03:40:27,532 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d52af7b428a94e31b40c0cca5bf6bd64 2024-11-11T03:40:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/cbf53d5eabc540a5bb30dcdeff546890 is 50, key is test_row_0/C:col10/1731296426513/Put/seqid=0 2024-11-11T03:40:27,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296487542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:27,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741987_1163 (size=12001) 2024-11-11T03:40:27,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/cbf53d5eabc540a5bb30dcdeff546890 2024-11-11T03:40:27,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/8464e4493f774d6d91b01436110bd54d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8464e4493f774d6d91b01436110bd54d 2024-11-11T03:40:27,577 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8464e4493f774d6d91b01436110bd54d, entries=150, sequenceid=117, filesize=30.2 K 2024-11-11T03:40:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d52af7b428a94e31b40c0cca5bf6bd64 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d52af7b428a94e31b40c0cca5bf6bd64 2024-11-11T03:40:27,584 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d52af7b428a94e31b40c0cca5bf6bd64, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T03:40:27,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/cbf53d5eabc540a5bb30dcdeff546890 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cbf53d5eabc540a5bb30dcdeff546890 2024-11-11T03:40:27,592 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cbf53d5eabc540a5bb30dcdeff546890, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T03:40:27,593 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 887247b757faef0e5f5ee791239fb929 in 590ms, sequenceid=117, compaction requested=false 2024-11-11T03:40:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T03:40:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T03:40:27,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T03:40:27,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2090 sec 2024-11-11T03:40:27,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2150 sec 2024-11-11T03:40:27,617 INFO [master/3ffe7f9d089e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T03:40:27,617 INFO [master/3ffe7f9d089e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T03:40:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:27,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:40:27,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:27,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:27,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:27,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:27,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:27,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:27,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111989dc6d81fbc41f68a0282fa732c1e7a_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296427223/Put/seqid=0 2024-11-11T03:40:27,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741988_1164 (size=14744) 2024-11-11T03:40:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296487920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:27,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:27,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296488032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:28,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296488252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:28,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111989dc6d81fbc41f68a0282fa732c1e7a_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111989dc6d81fbc41f68a0282fa732c1e7a_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:28,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/322a1dc9e52a488ea777c811281d2121, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:28,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/322a1dc9e52a488ea777c811281d2121 is 175, key is test_row_0/A:col10/1731296427223/Put/seqid=0 2024-11-11T03:40:28,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741989_1165 (size=39699) 2024-11-11T03:40:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T03:40:28,532 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:28,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T03:40:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T03:40:28,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:28,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:28,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296488561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T03:40:28,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T03:40:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:28,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,702 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/322a1dc9e52a488ea777c811281d2121 2024-11-11T03:40:28,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/921510fd85b643b68d7cfe0b3563f9a5 is 50, key is test_row_0/B:col10/1731296427223/Put/seqid=0 2024-11-11T03:40:28,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741990_1166 (size=12151) 2024-11-11T03:40:28,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T03:40:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:28,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T03:40:28,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296488931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:28,932 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296488942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:28,945 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296488952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:28,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296488952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:28,958 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,958 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 887247b757faef0e5f5ee791239fb929 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:28,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T03:40:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:28,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296489072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:29,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/921510fd85b643b68d7cfe0b3563f9a5 2024-11-11T03:40:29,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/cfcca77560b5440e8af1251214edd132 is 50, key is test_row_0/C:col10/1731296427223/Put/seqid=0 2024-11-11T03:40:29,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741991_1167 (size=12151) 2024-11-11T03:40:29,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T03:40:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T03:40:29,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T03:40:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:29,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T03:40:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:29,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/cfcca77560b5440e8af1251214edd132 2024-11-11T03:40:29,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/322a1dc9e52a488ea777c811281d2121 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/322a1dc9e52a488ea777c811281d2121 2024-11-11T03:40:29,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/322a1dc9e52a488ea777c811281d2121, entries=200, sequenceid=135, filesize=38.8 K 2024-11-11T03:40:29,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/921510fd85b643b68d7cfe0b3563f9a5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/921510fd85b643b68d7cfe0b3563f9a5 2024-11-11T03:40:29,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/921510fd85b643b68d7cfe0b3563f9a5, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T03:40:29,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/cfcca77560b5440e8af1251214edd132 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cfcca77560b5440e8af1251214edd132 2024-11-11T03:40:29,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cfcca77560b5440e8af1251214edd132, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T03:40:29,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 887247b757faef0e5f5ee791239fb929 in 1695ms, sequenceid=135, compaction requested=true 2024-11-11T03:40:29,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:29,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:29,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:29,550 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:29,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:29,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:29,550 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:29,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:29,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:29,551 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:29,551 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:29,552 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:29,552 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,552 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3be73bcffe53489882171a10e741a8cc, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d52af7b428a94e31b40c0cca5bf6bd64, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/921510fd85b643b68d7cfe0b3563f9a5] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.5 K 2024-11-11T03:40:29,554 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:29,554 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,554 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/dd86286032824d098431d0c2f021d31b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8464e4493f774d6d91b01436110bd54d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/322a1dc9e52a488ea777c811281d2121] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=99.4 K 2024-11-11T03:40:29,554 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,554 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/dd86286032824d098431d0c2f021d31b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8464e4493f774d6d91b01436110bd54d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/322a1dc9e52a488ea777c811281d2121] 2024-11-11T03:40:29,555 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3be73bcffe53489882171a10e741a8cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731296424747 2024-11-11T03:40:29,555 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd86286032824d098431d0c2f021d31b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731296424747 2024-11-11T03:40:29,555 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d52af7b428a94e31b40c0cca5bf6bd64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296426513 2024-11-11T03:40:29,555 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8464e4493f774d6d91b01436110bd54d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296426513 2024-11-11T03:40:29,555 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 921510fd85b643b68d7cfe0b3563f9a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296427216 2024-11-11T03:40:29,555 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 322a1dc9e52a488ea777c811281d2121, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296427200 2024-11-11T03:40:29,562 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:29,565 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111334dd73bff2749fc9c3585d12418f0b6_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:29,565 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:29,566 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/0a6c734b6a93497182bb9c3b829671c4 is 50, key is test_row_0/B:col10/1731296427223/Put/seqid=0 2024-11-11T03:40:29,567 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111334dd73bff2749fc9c3585d12418f0b6_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:29,568 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111334dd73bff2749fc9c3585d12418f0b6_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:29,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741993_1169 (size=4469) 2024-11-11T03:40:29,581 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#143 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:29,582 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0b5667bca59c4060932fab042f356419 is 175, key is test_row_0/A:col10/1731296427223/Put/seqid=0 2024-11-11T03:40:29,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741992_1168 (size=12459) 2024-11-11T03:40:29,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741994_1170 (size=31413) 2024-11-11T03:40:29,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T03:40:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:29,605 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:40:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ea104b17cc1476398894086b6e52488_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296427910/Put/seqid=0 2024-11-11T03:40:29,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741995_1171 (size=12304) 2024-11-11T03:40:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T03:40:29,998 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/0a6c734b6a93497182bb9c3b829671c4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/0a6c734b6a93497182bb9c3b829671c4 2024-11-11T03:40:30,004 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into 0a6c734b6a93497182bb9c3b829671c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:30,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:30,004 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=13, startTime=1731296429550; duration=0sec 2024-11-11T03:40:30,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:30,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:30,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:30,005 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:30,005 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:30,006 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:30,006 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/a17a7de85fe64585801da8604f385064, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cbf53d5eabc540a5bb30dcdeff546890, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cfcca77560b5440e8af1251214edd132] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.5 K 2024-11-11T03:40:30,007 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting a17a7de85fe64585801da8604f385064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731296424747 2024-11-11T03:40:30,007 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting cbf53d5eabc540a5bb30dcdeff546890, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296426513 2024-11-11T03:40:30,007 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting cfcca77560b5440e8af1251214edd132, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296427216 2024-11-11T03:40:30,010 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0b5667bca59c4060932fab042f356419 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0b5667bca59c4060932fab042f356419 2024-11-11T03:40:30,015 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into 0b5667bca59c4060932fab042f356419(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:30,015 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:30,015 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=13, startTime=1731296429550; duration=0sec 2024-11-11T03:40:30,015 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:30,016 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:30,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:30,022 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:30,024 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/e3d2a7b06e104577a82d997fab2392ed is 50, key is test_row_0/C:col10/1731296427223/Put/seqid=0 2024-11-11T03:40:30,025 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ea104b17cc1476398894086b6e52488_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ea104b17cc1476398894086b6e52488_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:30,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e60a577d0e9d41c98673dfd645a81c62, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:30,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e60a577d0e9d41c98673dfd645a81c62 is 175, key is test_row_0/A:col10/1731296427910/Put/seqid=0 2024-11-11T03:40:30,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741996_1172 (size=12459) 2024-11-11T03:40:30,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741997_1173 (size=31105) 2024-11-11T03:40:30,055 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e60a577d0e9d41c98673dfd645a81c62 2024-11-11T03:40:30,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/c145fddc3a34460baadf403b62ba12b4 is 50, key is test_row_0/B:col10/1731296427910/Put/seqid=0 2024-11-11T03:40:30,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741998_1174 (size=12151) 2024-11-11T03:40:30,073 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/c145fddc3a34460baadf403b62ba12b4 2024-11-11T03:40:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:30,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/4789111e644846caad1f0fac9af84caa is 50, key is test_row_0/C:col10/1731296427910/Put/seqid=0 2024-11-11T03:40:30,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741999_1175 (size=12151) 2024-11-11T03:40:30,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296490138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:30,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296490251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:30,442 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/e3d2a7b06e104577a82d997fab2392ed as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/e3d2a7b06e104577a82d997fab2392ed 2024-11-11T03:40:30,447 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into e3d2a7b06e104577a82d997fab2392ed(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:30,447 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:30,447 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=13, startTime=1731296429550; duration=0sec 2024-11-11T03:40:30,447 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:30,447 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:30,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296490462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:30,497 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/4789111e644846caad1f0fac9af84caa 2024-11-11T03:40:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e60a577d0e9d41c98673dfd645a81c62 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e60a577d0e9d41c98673dfd645a81c62 2024-11-11T03:40:30,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e60a577d0e9d41c98673dfd645a81c62, entries=150, sequenceid=152, filesize=30.4 K 2024-11-11T03:40:30,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/c145fddc3a34460baadf403b62ba12b4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/c145fddc3a34460baadf403b62ba12b4 2024-11-11T03:40:30,512 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/c145fddc3a34460baadf403b62ba12b4, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T03:40:30,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/4789111e644846caad1f0fac9af84caa as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/4789111e644846caad1f0fac9af84caa 2024-11-11T03:40:30,517 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/4789111e644846caad1f0fac9af84caa, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T03:40:30,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 887247b757faef0e5f5ee791239fb929 in 913ms, sequenceid=152, compaction requested=false 2024-11-11T03:40:30,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:30,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:30,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T03:40:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T03:40:30,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T03:40:30,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9830 sec 2024-11-11T03:40:30,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9880 sec 2024-11-11T03:40:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T03:40:30,682 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:30,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T03:40:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T03:40:30,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:30,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:30,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:30,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T03:40:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:30,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111129a66bfbacde4e70a67e6a0435fa72b7_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296430136/Put/seqid=0 2024-11-11T03:40:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T03:40:30,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742000_1176 (size=14794) 2024-11-11T03:40:30,807 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:30,812 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111129a66bfbacde4e70a67e6a0435fa72b7_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111129a66bfbacde4e70a67e6a0435fa72b7_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:30,813 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/59cda0c847424f3dacd7e7ddc8f886e0, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:30,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/59cda0c847424f3dacd7e7ddc8f886e0 is 175, key is test_row_0/A:col10/1731296430136/Put/seqid=0 2024-11-11T03:40:30,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742001_1177 (size=39749) 2024-11-11T03:40:30,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:30,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:30,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:30,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:30,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:30,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296490846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:30,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296490961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:30,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:30,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:30,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:30,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T03:40:31,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:31,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:31,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296491172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:31,233 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/59cda0c847424f3dacd7e7ddc8f886e0 2024-11-11T03:40:31,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/56d5e9f0d4bb41aabac401af2bb43cda is 50, key is test_row_0/B:col10/1731296430136/Put/seqid=0 2024-11-11T03:40:31,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742002_1178 (size=12151) 2024-11-11T03:40:31,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T03:40:31,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:31,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296491481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:31,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/56d5e9f0d4bb41aabac401af2bb43cda 2024-11-11T03:40:31,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/329437a494194265869181fda9347fcc is 50, key is test_row_0/C:col10/1731296430136/Put/seqid=0 2024-11-11T03:40:31,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742003_1179 (size=12151) 2024-11-11T03:40:31,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T03:40:31,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:31,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:31,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296491992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:32,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:32,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/329437a494194265869181fda9347fcc 2024-11-11T03:40:32,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/59cda0c847424f3dacd7e7ddc8f886e0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/59cda0c847424f3dacd7e7ddc8f886e0 2024-11-11T03:40:32,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/59cda0c847424f3dacd7e7ddc8f886e0, entries=200, sequenceid=171, filesize=38.8 K 2024-11-11T03:40:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/56d5e9f0d4bb41aabac401af2bb43cda as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/56d5e9f0d4bb41aabac401af2bb43cda 2024-11-11T03:40:32,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/56d5e9f0d4bb41aabac401af2bb43cda, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T03:40:32,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/329437a494194265869181fda9347fcc as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/329437a494194265869181fda9347fcc 2024-11-11T03:40:32,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/329437a494194265869181fda9347fcc, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T03:40:32,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 887247b757faef0e5f5ee791239fb929 in 1307ms, sequenceid=171, compaction requested=true 2024-11-11T03:40:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:32,082 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:32,082 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:32,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:32,083 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:32,083 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:32,083 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:32,083 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:32,083 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,083 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,083 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0b5667bca59c4060932fab042f356419, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e60a577d0e9d41c98673dfd645a81c62, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/59cda0c847424f3dacd7e7ddc8f886e0] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=99.9 K 2024-11-11T03:40:32,083 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/0a6c734b6a93497182bb9c3b829671c4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/c145fddc3a34460baadf403b62ba12b4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/56d5e9f0d4bb41aabac401af2bb43cda] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.9 K 2024-11-11T03:40:32,083 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,083 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0b5667bca59c4060932fab042f356419, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e60a577d0e9d41c98673dfd645a81c62, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/59cda0c847424f3dacd7e7ddc8f886e0] 2024-11-11T03:40:32,084 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6c734b6a93497182bb9c3b829671c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296427216 2024-11-11T03:40:32,084 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5667bca59c4060932fab042f356419, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296427216 2024-11-11T03:40:32,084 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting c145fddc3a34460baadf403b62ba12b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731296427907 2024-11-11T03:40:32,084 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60a577d0e9d41c98673dfd645a81c62, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731296427907 2024-11-11T03:40:32,084 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59cda0c847424f3dacd7e7ddc8f886e0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731296430108 2024-11-11T03:40:32,084 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 56d5e9f0d4bb41aabac401af2bb43cda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731296430108 2024-11-11T03:40:32,091 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:32,091 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:32,092 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d3715962cf1c4a7bb3844fd3a23470fe is 50, key is test_row_0/B:col10/1731296430136/Put/seqid=0 2024-11-11T03:40:32,103 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411111a9ead9a54824678a20a4a6bd6fd9e48_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:32,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742004_1180 (size=12561) 2024-11-11T03:40:32,105 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411111a9ead9a54824678a20a4a6bd6fd9e48_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:32,105 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111a9ead9a54824678a20a4a6bd6fd9e48_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:32,110 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/d3715962cf1c4a7bb3844fd3a23470fe as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d3715962cf1c4a7bb3844fd3a23470fe 2024-11-11T03:40:32,116 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into d3715962cf1c4a7bb3844fd3a23470fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:32,117 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:32,117 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=13, startTime=1731296432082; duration=0sec 2024-11-11T03:40:32,117 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:32,117 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:32,117 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:32,118 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:32,118 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:32,118 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,118 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/e3d2a7b06e104577a82d997fab2392ed, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/4789111e644846caad1f0fac9af84caa, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/329437a494194265869181fda9347fcc] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=35.9 K 2024-11-11T03:40:32,119 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d2a7b06e104577a82d997fab2392ed, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296427216 2024-11-11T03:40:32,119 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4789111e644846caad1f0fac9af84caa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731296427907 2024-11-11T03:40:32,119 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 329437a494194265869181fda9347fcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731296430108 2024-11-11T03:40:32,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742005_1181 (size=4469) 2024-11-11T03:40:32,122 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#153 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:32,123 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/3d78ca7f34b84626855760754b305b5c is 175, key is test_row_0/A:col10/1731296430136/Put/seqid=0 2024-11-11T03:40:32,132 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:32,132 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/b7e3f71238164ada92389e8b704adec1 is 50, key is test_row_0/C:col10/1731296430136/Put/seqid=0 2024-11-11T03:40:32,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742006_1182 (size=31515) 2024-11-11T03:40:32,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742007_1183 (size=12561) 2024-11-11T03:40:32,142 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/3d78ca7f34b84626855760754b305b5c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/3d78ca7f34b84626855760754b305b5c 2024-11-11T03:40:32,146 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into 3d78ca7f34b84626855760754b305b5c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:32,146 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:32,146 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=13, startTime=1731296432082; duration=0sec 2024-11-11T03:40:32,146 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:32,147 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:32,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T03:40:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,212 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T03:40:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b679fe2cae14a53bf1e57b927ef10fb_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296430833/Put/seqid=0 2024-11-11T03:40:32,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742008_1184 (size=12304) 2024-11-11T03:40:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,234 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b679fe2cae14a53bf1e57b927ef10fb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b679fe2cae14a53bf1e57b927ef10fb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:32,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/65ac8dbd14ba4bdcb20214d4ee277477, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:32,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/65ac8dbd14ba4bdcb20214d4ee277477 is 175, key is test_row_0/A:col10/1731296430833/Put/seqid=0 2024-11-11T03:40:32,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742009_1185 (size=31105) 2024-11-11T03:40:32,543 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/b7e3f71238164ada92389e8b704adec1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b7e3f71238164ada92389e8b704adec1 2024-11-11T03:40:32,549 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into b7e3f71238164ada92389e8b704adec1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:32,549 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:32,549 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=13, startTime=1731296432082; duration=0sec 2024-11-11T03:40:32,549 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:32,549 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:32,651 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/65ac8dbd14ba4bdcb20214d4ee277477 2024-11-11T03:40:32,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/9f2e028fa6374ab88e062ba8f2d32843 is 50, key is test_row_0/B:col10/1731296430833/Put/seqid=0 2024-11-11T03:40:32,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742010_1186 (size=12151) 2024-11-11T03:40:32,673 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/9f2e028fa6374ab88e062ba8f2d32843 2024-11-11T03:40:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/b86b09e893de4b048d31d9811bb5b883 is 50, key is test_row_0/C:col10/1731296430833/Put/seqid=0 2024-11-11T03:40:32,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742011_1187 (size=12151) 2024-11-11T03:40:32,694 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/b86b09e893de4b048d31d9811bb5b883 2024-11-11T03:40:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/65ac8dbd14ba4bdcb20214d4ee277477 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/65ac8dbd14ba4bdcb20214d4ee277477 2024-11-11T03:40:32,703 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/65ac8dbd14ba4bdcb20214d4ee277477, entries=150, sequenceid=189, filesize=30.4 K 2024-11-11T03:40:32,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/9f2e028fa6374ab88e062ba8f2d32843 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9f2e028fa6374ab88e062ba8f2d32843 2024-11-11T03:40:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,709 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9f2e028fa6374ab88e062ba8f2d32843, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T03:40:32,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/b86b09e893de4b048d31d9811bb5b883 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b86b09e893de4b048d31d9811bb5b883 2024-11-11T03:40:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,725 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b86b09e893de4b048d31d9811bb5b883, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T03:40:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,726 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 887247b757faef0e5f5ee791239fb929 in 514ms, sequenceid=189, compaction requested=false 2024-11-11T03:40:32,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:32,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T03:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T03:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T03:40:32,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0440 sec 2024-11-11T03:40:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0490 sec 2024-11-11T03:40:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T03:40:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,832 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T03:40:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T03:40:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,835 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,836 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T03:40:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T03:40:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:32,989 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-11T03:40:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:32,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:32,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:32,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:32,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119be260bb394d49b9adcc5ac8ac8c0590_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_1/A:col10/1731296432971/Put/seqid=0 2024-11-11T03:40:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742012_1188 (size=7324) 2024-11-11T03:40:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,010 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119be260bb394d49b9adcc5ac8ac8c0590_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119be260bb394d49b9adcc5ac8ac8c0590_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/fa57d22b9fed43e7919ad627a960a3c8, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/fa57d22b9fed43e7919ad627a960a3c8 is 175, key is test_row_1/A:col10/1731296432971/Put/seqid=0 2024-11-11T03:40:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742013_1189 (size=13815) 2024-11-11T03:40:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,025 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/fa57d22b9fed43e7919ad627a960a3c8 2024-11-11T03:40:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/038c3b1d9e9b4e76b2a309c44b724f1e is 50, key is test_row_1/B:col10/1731296432971/Put/seqid=0 2024-11-11T03:40:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742014_1190 (size=7365) 2024-11-11T03:40:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,044 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/038c3b1d9e9b4e76b2a309c44b724f1e 2024-11-11T03:40:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:33,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/fa8b4008ac3544588983756f1d6496ce is 50, key is test_row_1/C:col10/1731296432971/Put/seqid=0 2024-11-11T03:40:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742015_1191 (size=7365) 2024-11-11T03:40:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,070 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/fa8b4008ac3544588983756f1d6496ce 2024-11-11T03:40:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/fa57d22b9fed43e7919ad627a960a3c8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/fa57d22b9fed43e7919ad627a960a3c8 2024-11-11T03:40:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/fa57d22b9fed43e7919ad627a960a3c8, entries=50, sequenceid=194, filesize=13.5 K 2024-11-11T03:40:33,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/038c3b1d9e9b4e76b2a309c44b724f1e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/038c3b1d9e9b4e76b2a309c44b724f1e 2024-11-11T03:40:33,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731296433060, max timestamp=1731296433085Num uniques 859; 2024-11-11T03:40:33,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T03:40:33,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731296433060, max timestamp=1731296433085Num uniques 859; 2024-11-11T03:40:33,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T03:40:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731296433060, max timestamp=1731296433085Num uniques 859; 2024-11-11T03:40:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T03:40:33,097 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/038c3b1d9e9b4e76b2a309c44b724f1e, entries=50, sequenceid=194, filesize=7.2 K 2024-11-11T03:40:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/fa8b4008ac3544588983756f1d6496ce as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/fa8b4008ac3544588983756f1d6496ce 2024-11-11T03:40:33,105 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/fa8b4008ac3544588983756f1d6496ce, entries=50, sequenceid=194, filesize=7.2 K 2024-11-11T03:40:33,106 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=80.51 KB/82440 for 887247b757faef0e5f5ee791239fb929 in 117ms, sequenceid=194, compaction requested=true 2024-11-11T03:40:33,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:33,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T03:40:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T03:40:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:33,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=93.93 KB heapSize=251.39 KB 2024-11-11T03:40:33,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:33,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:33,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:33,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:33,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T03:40:33,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 272 msec 2024-11-11T03:40:33,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 280 msec 2024-11-11T03:40:33,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cd493350eee44270b4878567cbbe9494_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296433100/Put/seqid=0 2024-11-11T03:40:33,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742016_1192 (size=12304) 2024-11-11T03:40:33,137 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cd493350eee44270b4878567cbbe9494_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cd493350eee44270b4878567cbbe9494_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:33,146 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e6f4a28f5c4741ee89a903d80b5654fb, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:33,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e6f4a28f5c4741ee89a903d80b5654fb is 175, key is test_row_0/A:col10/1731296433100/Put/seqid=0 2024-11-11T03:40:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T03:40:33,152 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:33,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T03:40:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T03:40:33,155 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:33,156 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:33,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:33,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296493146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296493153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742017_1193 (size=31105) 2024-11-11T03:40:33,164 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e6f4a28f5c4741ee89a903d80b5654fb 2024-11-11T03:40:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296493157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296493157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/937e6c11893448c0b8a2185dc53dc42e is 50, key is test_row_0/B:col10/1731296433100/Put/seqid=0 2024-11-11T03:40:33,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296493160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742018_1194 (size=12151) 2024-11-11T03:40:33,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/937e6c11893448c0b8a2185dc53dc42e 2024-11-11T03:40:33,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/859e10e3816c4456843f0e74c4a07100 is 50, key is test_row_0/C:col10/1731296433100/Put/seqid=0 2024-11-11T03:40:33,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742019_1195 (size=12151) 2024-11-11T03:40:33,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/859e10e3816c4456843f0e74c4a07100 2024-11-11T03:40:33,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/e6f4a28f5c4741ee89a903d80b5654fb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e6f4a28f5c4741ee89a903d80b5654fb 2024-11-11T03:40:33,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e6f4a28f5c4741ee89a903d80b5654fb, entries=150, sequenceid=222, filesize=30.4 K 2024-11-11T03:40:33,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/937e6c11893448c0b8a2185dc53dc42e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/937e6c11893448c0b8a2185dc53dc42e 2024-11-11T03:40:33,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/937e6c11893448c0b8a2185dc53dc42e, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T03:40:33,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/859e10e3816c4456843f0e74c4a07100 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/859e10e3816c4456843f0e74c4a07100 2024-11-11T03:40:33,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/859e10e3816c4456843f0e74c4a07100, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T03:40:33,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~251.34 KB/257376, currentSize=100.63 KB/103050 for 887247b757faef0e5f5ee791239fb929 in 139ms, sequenceid=222, compaction requested=true 2024-11-11T03:40:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:33,247 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:33,247 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:33,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:33,249 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 107540 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:33,249 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44228 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:33,249 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:33,249 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:33,249 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,249 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,249 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/3d78ca7f34b84626855760754b305b5c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/65ac8dbd14ba4bdcb20214d4ee277477, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/fa57d22b9fed43e7919ad627a960a3c8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e6f4a28f5c4741ee89a903d80b5654fb] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=105.0 K 2024-11-11T03:40:33,250 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d3715962cf1c4a7bb3844fd3a23470fe, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9f2e028fa6374ab88e062ba8f2d32843, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/038c3b1d9e9b4e76b2a309c44b724f1e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/937e6c11893448c0b8a2185dc53dc42e] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=43.2 K 2024-11-11T03:40:33,250 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,250 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/3d78ca7f34b84626855760754b305b5c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/65ac8dbd14ba4bdcb20214d4ee277477, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/fa57d22b9fed43e7919ad627a960a3c8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e6f4a28f5c4741ee89a903d80b5654fb] 2024-11-11T03:40:33,250 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d3715962cf1c4a7bb3844fd3a23470fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731296430108 2024-11-11T03:40:33,250 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d78ca7f34b84626855760754b305b5c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731296430108 2024-11-11T03:40:33,250 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2e028fa6374ab88e062ba8f2d32843, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296430802 2024-11-11T03:40:33,251 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ac8dbd14ba4bdcb20214d4ee277477, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296430802 2024-11-11T03:40:33,251 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 038c3b1d9e9b4e76b2a309c44b724f1e, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296432971 2024-11-11T03:40:33,251 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa57d22b9fed43e7919ad627a960a3c8, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296432971 2024-11-11T03:40:33,251 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 937e6c11893448c0b8a2185dc53dc42e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731296433100 2024-11-11T03:40:33,251 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f4a28f5c4741ee89a903d80b5654fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731296433100 2024-11-11T03:40:33,260 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:33,263 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:33,263 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/41a1a24edaaf47f9bb61f38425ad889f is 50, key is test_row_0/B:col10/1731296433100/Put/seqid=0 2024-11-11T03:40:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T03:40:33,266 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ab3ca91c3244403f96fe7a180ca61c42_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:33,268 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ab3ca91c3244403f96fe7a180ca61c42_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:33,268 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ab3ca91c3244403f96fe7a180ca61c42_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:33,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T03:40:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296493285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296493286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742020_1196 (size=12697) 2024-11-11T03:40:33,298 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/41a1a24edaaf47f9bb61f38425ad889f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/41a1a24edaaf47f9bb61f38425ad889f 2024-11-11T03:40:33,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296493288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296493289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296493290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,305 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into 41a1a24edaaf47f9bb61f38425ad889f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:33,305 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:33,305 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=12, startTime=1731296433247; duration=0sec 2024-11-11T03:40:33,306 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:33,306 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:33,306 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:33,308 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44228 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:33,308 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:33,308 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,308 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b7e3f71238164ada92389e8b704adec1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b86b09e893de4b048d31d9811bb5b883, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/fa8b4008ac3544588983756f1d6496ce, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/859e10e3816c4456843f0e74c4a07100] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=43.2 K 2024-11-11T03:40:33,309 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e3f71238164ada92389e8b704adec1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731296430108 2024-11-11T03:40:33,309 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b86b09e893de4b048d31d9811bb5b883, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296430802 2024-11-11T03:40:33,310 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting fa8b4008ac3544588983756f1d6496ce, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731296432971 2024-11-11T03:40:33,310 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 859e10e3816c4456843f0e74c4a07100, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731296433100 2024-11-11T03:40:33,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:33,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:33,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e37e16363677465ab64f3b5531abd9d2_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296433144/Put/seqid=0 2024-11-11T03:40:33,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742021_1197 (size=4469) 2024-11-11T03:40:33,329 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:33,330 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/1abd50d4956643fcba6a72d321ff8d97 is 50, key is test_row_0/C:col10/1731296433100/Put/seqid=0 2024-11-11T03:40:33,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742023_1199 (size=12697) 2024-11-11T03:40:33,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742022_1198 (size=17284) 2024-11-11T03:40:33,350 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:33,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e37e16363677465ab64f3b5531abd9d2_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e37e16363677465ab64f3b5531abd9d2_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:33,359 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/ad76ea550c5d478e8d06777e1b5fcc03, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:33,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/ad76ea550c5d478e8d06777e1b5fcc03 is 175, key is test_row_0/A:col10/1731296433144/Put/seqid=0 2024-11-11T03:40:33,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742024_1200 (size=48389) 2024-11-11T03:40:33,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296493392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296493393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296493402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296493412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296493412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T03:40:33,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:33,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:33,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296493602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296493602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296493611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296493622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:33,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:33,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296493632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,726 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#164 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:33,726 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/58550c674b124f89ac3213eb726c10f1 is 175, key is test_row_0/A:col10/1731296433100/Put/seqid=0 2024-11-11T03:40:33,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742025_1201 (size=31651) 2024-11-11T03:40:33,751 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/1abd50d4956643fcba6a72d321ff8d97 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1abd50d4956643fcba6a72d321ff8d97 2024-11-11T03:40:33,755 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into 1abd50d4956643fcba6a72d321ff8d97(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:33,755 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:33,755 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=12, startTime=1731296433247; duration=0sec 2024-11-11T03:40:33,755 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:33,755 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:33,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:33,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:33,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T03:40:33,784 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/ad76ea550c5d478e8d06777e1b5fcc03 2024-11-11T03:40:33,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7fa7e2028ea94f3c879d3ef7f2378db9 is 50, key is test_row_0/B:col10/1731296433144/Put/seqid=0 2024-11-11T03:40:33,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742026_1202 (size=12151) 2024-11-11T03:40:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296493911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296493911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296493922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:33,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:33,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:33,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296493932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296493941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:34,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:34,136 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/58550c674b124f89ac3213eb726c10f1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/58550c674b124f89ac3213eb726c10f1 2024-11-11T03:40:34,143 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into 58550c674b124f89ac3213eb726c10f1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:34,143 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:34,143 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=12, startTime=1731296433247; duration=0sec 2024-11-11T03:40:34,143 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:34,143 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:34,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7fa7e2028ea94f3c879d3ef7f2378db9 2024-11-11T03:40:34,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/0875c7740568481ab8bfaf42b6a646ee is 50, key is test_row_0/C:col10/1731296433144/Put/seqid=0 2024-11-11T03:40:34,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742027_1203 (size=12151) 2024-11-11T03:40:34,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/0875c7740568481ab8bfaf42b6a646ee 2024-11-11T03:40:34,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/ad76ea550c5d478e8d06777e1b5fcc03 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/ad76ea550c5d478e8d06777e1b5fcc03 2024-11-11T03:40:34,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/ad76ea550c5d478e8d06777e1b5fcc03, entries=250, sequenceid=242, filesize=47.3 K 2024-11-11T03:40:34,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7fa7e2028ea94f3c879d3ef7f2378db9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7fa7e2028ea94f3c879d3ef7f2378db9 2024-11-11T03:40:34,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7fa7e2028ea94f3c879d3ef7f2378db9, entries=150, sequenceid=242, filesize=11.9 K 2024-11-11T03:40:34,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/0875c7740568481ab8bfaf42b6a646ee as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/0875c7740568481ab8bfaf42b6a646ee 2024-11-11T03:40:34,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/0875c7740568481ab8bfaf42b6a646ee, entries=150, sequenceid=242, filesize=11.9 K 2024-11-11T03:40:34,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:34,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 887247b757faef0e5f5ee791239fb929 in 972ms, sequenceid=242, compaction requested=false 2024-11-11T03:40:34,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:34,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:34,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T03:40:34,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T03:40:34,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:34,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T03:40:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113492de9592674351a82bce5c748697ab_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296433285/Put/seqid=0 2024-11-11T03:40:34,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742028_1204 (size=12304) 2024-11-11T03:40:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:34,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296494453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296494454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296494456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296494458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296494457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296494562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296494562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296494562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296494571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296494571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296494786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296494788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296494788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296494788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296494788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:34,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:34,812 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113492de9592674351a82bce5c748697ab_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113492de9592674351a82bce5c748697ab_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:34,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/439c3a1445f146e9ba083894e0584169, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/439c3a1445f146e9ba083894e0584169 is 175, key is test_row_0/A:col10/1731296433285/Put/seqid=0 2024-11-11T03:40:34,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742029_1205 (size=31105) 2024-11-11T03:40:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296495102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296495102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296495102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296495102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296495102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,218 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/439c3a1445f146e9ba083894e0584169 2024-11-11T03:40:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/44c8ce1b6a1e4c30af243c7342c496c0 is 50, key is test_row_0/B:col10/1731296433285/Put/seqid=0 2024-11-11T03:40:35,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742030_1206 (size=12151) 2024-11-11T03:40:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T03:40:35,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296495612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296495612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296495612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296495613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296495613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:35,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/44c8ce1b6a1e4c30af243c7342c496c0 2024-11-11T03:40:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/69dec9fe52dd41daa8987e8071413620 is 50, key is test_row_0/C:col10/1731296433285/Put/seqid=0 2024-11-11T03:40:35,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742031_1207 (size=12151) 2024-11-11T03:40:35,643 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/69dec9fe52dd41daa8987e8071413620 2024-11-11T03:40:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/439c3a1445f146e9ba083894e0584169 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/439c3a1445f146e9ba083894e0584169 2024-11-11T03:40:35,651 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/439c3a1445f146e9ba083894e0584169, entries=150, sequenceid=257, filesize=30.4 K 2024-11-11T03:40:35,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/44c8ce1b6a1e4c30af243c7342c496c0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/44c8ce1b6a1e4c30af243c7342c496c0 2024-11-11T03:40:35,656 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/44c8ce1b6a1e4c30af243c7342c496c0, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T03:40:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/69dec9fe52dd41daa8987e8071413620 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/69dec9fe52dd41daa8987e8071413620 2024-11-11T03:40:35,661 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/69dec9fe52dd41daa8987e8071413620, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T03:40:35,662 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 887247b757faef0e5f5ee791239fb929 in 1270ms, sequenceid=257, compaction requested=true 2024-11-11T03:40:35,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:35,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:35,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T03:40:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T03:40:35,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T03:40:35,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5090 sec 2024-11-11T03:40:35,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.5160 sec 2024-11-11T03:40:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:36,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T03:40:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:36,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dd150e2e89a2425c94c19f3208f1a4fb_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296436626/Put/seqid=0 2024-11-11T03:40:36,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742032_1208 (size=17534) 2024-11-11T03:40:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296496634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296496635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296496636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296496637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296496638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296496752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296496752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296496752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296496752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296496752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:36,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,039 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:37,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dd150e2e89a2425c94c19f3208f1a4fb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dd150e2e89a2425c94c19f3208f1a4fb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:37,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/31638a8fb8b0462a8d12a9e10497e90a, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:37,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/31638a8fb8b0462a8d12a9e10497e90a is 175, key is test_row_0/A:col10/1731296436626/Put/seqid=0 2024-11-11T03:40:37,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742033_1209 (size=48639) 2024-11-11T03:40:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296497272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296497272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296497272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296497272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296497272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T03:40:37,312 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:37,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T03:40:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T03:40:37,314 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:37,315 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:37,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T03:40:37,449 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/31638a8fb8b0462a8d12a9e10497e90a 2024-11-11T03:40:37,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3208d721bcd1417e9b8c5a2f112813e1 is 50, key is test_row_0/B:col10/1731296436626/Put/seqid=0 2024-11-11T03:40:37,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742034_1210 (size=12301) 2024-11-11T03:40:37,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T03:40:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:37,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T03:40:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T03:40:37,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T03:40:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:37,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296497782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296497782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296497792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296497792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296497792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:37,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:37,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3208d721bcd1417e9b8c5a2f112813e1 2024-11-11T03:40:37,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/05a678a1618b41afaec2bba3edc57fd9 is 50, key is test_row_0/C:col10/1731296436626/Put/seqid=0 2024-11-11T03:40:37,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742035_1211 (size=12301) 2024-11-11T03:40:37,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T03:40:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:37,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T03:40:38,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T03:40:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:38,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T03:40:38,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:38,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/05a678a1618b41afaec2bba3edc57fd9 2024-11-11T03:40:38,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/31638a8fb8b0462a8d12a9e10497e90a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/31638a8fb8b0462a8d12a9e10497e90a 2024-11-11T03:40:38,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/31638a8fb8b0462a8d12a9e10497e90a, entries=250, sequenceid=279, filesize=47.5 K 2024-11-11T03:40:38,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/3208d721bcd1417e9b8c5a2f112813e1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3208d721bcd1417e9b8c5a2f112813e1 2024-11-11T03:40:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3208d721bcd1417e9b8c5a2f112813e1, entries=150, sequenceid=279, filesize=12.0 K 2024-11-11T03:40:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/05a678a1618b41afaec2bba3edc57fd9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/05a678a1618b41afaec2bba3edc57fd9 2024-11-11T03:40:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/05a678a1618b41afaec2bba3edc57fd9, entries=150, sequenceid=279, filesize=12.0 K 2024-11-11T03:40:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 887247b757faef0e5f5ee791239fb929 in 1671ms, sequenceid=279, compaction requested=true 2024-11-11T03:40:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:38,298 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:38,298 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,300 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,300 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:38,300 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:38,300 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:38,300 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,300 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,300 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/41a1a24edaaf47f9bb61f38425ad889f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7fa7e2028ea94f3c879d3ef7f2378db9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/44c8ce1b6a1e4c30af243c7342c496c0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3208d721bcd1417e9b8c5a2f112813e1] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=48.1 K 2024-11-11T03:40:38,300 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/58550c674b124f89ac3213eb726c10f1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/ad76ea550c5d478e8d06777e1b5fcc03, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/439c3a1445f146e9ba083894e0584169, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/31638a8fb8b0462a8d12a9e10497e90a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=156.0 K 2024-11-11T03:40:38,300 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,300 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/58550c674b124f89ac3213eb726c10f1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/ad76ea550c5d478e8d06777e1b5fcc03, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/439c3a1445f146e9ba083894e0584169, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/31638a8fb8b0462a8d12a9e10497e90a] 2024-11-11T03:40:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,300 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a1a24edaaf47f9bb61f38425ad889f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731296433100 2024-11-11T03:40:38,300 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58550c674b124f89ac3213eb726c10f1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731296433100 2024-11-11T03:40:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,301 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad76ea550c5d478e8d06777e1b5fcc03, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731296433144 2024-11-11T03:40:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,301 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa7e2028ea94f3c879d3ef7f2378db9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731296433144 2024-11-11T03:40:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,301 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439c3a1445f146e9ba083894e0584169, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731296433285 2024-11-11T03:40:38,301 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c8ce1b6a1e4c30af243c7342c496c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731296433285 2024-11-11T03:40:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,302 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31638a8fb8b0462a8d12a9e10497e90a, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731296434445 2024-11-11T03:40:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,302 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3208d721bcd1417e9b8c5a2f112813e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731296434445 2024-11-11T03:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,313 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,316 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,316 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/b4d3b7cbffe543ff840cdec1c3fc372c is 50, key is test_row_0/B:col10/1731296436626/Put/seqid=0 2024-11-11T03:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,345 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117aa9b117d62446fabab2b4a6dd7190dc_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,348 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117aa9b117d62446fabab2b4a6dd7190dc_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:38,348 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117aa9b117d62446fabab2b4a6dd7190dc_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742036_1212 (size=12983) 2024-11-11T03:40:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742037_1213 (size=4469) 2024-11-11T03:40:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,378 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/b4d3b7cbffe543ff840cdec1c3fc372c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/b4d3b7cbffe543ff840cdec1c3fc372c 2024-11-11T03:40:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T03:40:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,383 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:40:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,385 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into b4d3b7cbffe543ff840cdec1c3fc372c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:38,385 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:38,385 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=12, startTime=1731296438298; duration=0sec 2024-11-11T03:40:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,385 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:38,385 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:38,385 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,388 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:38,388 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,388 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,389 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1abd50d4956643fcba6a72d321ff8d97, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/0875c7740568481ab8bfaf42b6a646ee, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/69dec9fe52dd41daa8987e8071413620, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/05a678a1618b41afaec2bba3edc57fd9] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=48.1 K 2024-11-11T03:40:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,389 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abd50d4956643fcba6a72d321ff8d97, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731296433100 2024-11-11T03:40:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,390 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0875c7740568481ab8bfaf42b6a646ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731296433144 2024-11-11T03:40:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,390 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 69dec9fe52dd41daa8987e8071413620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731296433285 2024-11-11T03:40:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,391 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a678a1618b41afaec2bba3edc57fd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731296434445 2024-11-11T03:40:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d0e118a69bf1489490b023e1a4a33705_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_1/A:col10/1731296436637/Put/seqid=0 2024-11-11T03:40:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742038_1214 (size=9914) 2024-11-11T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,413 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,414 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/6805ba34788a443987aa1531eb140fc7 is 50, key is test_row_0/C:col10/1731296436626/Put/seqid=0 2024-11-11T03:40:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742039_1215 (size=12983) 2024-11-11T03:40:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T03:40:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,779 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#176 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,780 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/eceed955dab84c4db55480bd2fc47c89 is 175, key is test_row_0/A:col10/1731296436626/Put/seqid=0 2024-11-11T03:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742040_1216 (size=31937) 2024-11-11T03:40:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,796 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/eceed955dab84c4db55480bd2fc47c89 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/eceed955dab84c4db55480bd2fc47c89 2024-11-11T03:40:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,803 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into eceed955dab84c4db55480bd2fc47c89(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:38,803 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:38,803 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=12, startTime=1731296438298; duration=0sec 2024-11-11T03:40:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,803 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:38,803 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,813 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d0e118a69bf1489490b023e1a4a33705_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d0e118a69bf1489490b023e1a4a33705_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f8112cd31cf74db9a80665a57dd0a7d0, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f8112cd31cf74db9a80665a57dd0a7d0 is 175, key is test_row_1/A:col10/1731296436637/Put/seqid=0 2024-11-11T03:40:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742041_1217 (size=22561) 2024-11-11T03:40:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,838 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f8112cd31cf74db9a80665a57dd0a7d0 2024-11-11T03:40:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:38,843 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/6805ba34788a443987aa1531eb140fc7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6805ba34788a443987aa1531eb140fc7 2024-11-11T03:40:38,854 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into 6805ba34788a443987aa1531eb140fc7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:38,854 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:38,854 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=12, startTime=1731296438298; duration=0sec 2024-11-11T03:40:38,854 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:38,854 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4294fbe8d0cc46839400de947158f08b is 50, key is test_row_1/B:col10/1731296436637/Put/seqid=0 2024-11-11T03:40:38,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731296438847, max timestamp=1731296438859Num uniques 859; 2024-11-11T03:40:38,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T03:40:38,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731296438847, max timestamp=1731296438859Num uniques 859; 2024-11-11T03:40:38,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T03:40:38,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731296438847, max timestamp=1731296438859Num uniques 859; 2024-11-11T03:40:38,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T03:40:38,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742042_1218 (size=9857) 2024-11-11T03:40:38,884 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4294fbe8d0cc46839400de947158f08b 2024-11-11T03:40:38,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296498881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:38,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:38,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296498886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296498886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296498888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296498890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:38,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/72a3a570035f4352a318597e3adf9d27 is 50, key is test_row_1/C:col10/1731296436637/Put/seqid=0 2024-11-11T03:40:38,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742043_1219 (size=9857) 2024-11-11T03:40:38,913 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/72a3a570035f4352a318597e3adf9d27 2024-11-11T03:40:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f8112cd31cf74db9a80665a57dd0a7d0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f8112cd31cf74db9a80665a57dd0a7d0 2024-11-11T03:40:38,927 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f8112cd31cf74db9a80665a57dd0a7d0, entries=100, sequenceid=289, filesize=22.0 K 2024-11-11T03:40:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4294fbe8d0cc46839400de947158f08b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4294fbe8d0cc46839400de947158f08b 2024-11-11T03:40:38,932 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4294fbe8d0cc46839400de947158f08b, entries=100, sequenceid=289, filesize=9.6 K 2024-11-11T03:40:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/72a3a570035f4352a318597e3adf9d27 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/72a3a570035f4352a318597e3adf9d27 2024-11-11T03:40:38,937 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/72a3a570035f4352a318597e3adf9d27, entries=100, sequenceid=289, filesize=9.6 K 2024-11-11T03:40:38,940 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 887247b757faef0e5f5ee791239fb929 in 557ms, sequenceid=289, compaction requested=false 2024-11-11T03:40:38,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:38,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:38,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T03:40:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T03:40:38,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T03:40:38,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6270 sec 2024-11-11T03:40:38,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6350 sec 2024-11-11T03:40:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:38,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-11T03:40:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:39,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:39,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a37e1304c9c44d948f77809213cb4977_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296438876/Put/seqid=0 2024-11-11T03:40:39,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296499006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296499006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296499007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296499007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296499008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742044_1220 (size=14994) 2024-11-11T03:40:39,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a37e1304c9c44d948f77809213cb4977_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a37e1304c9c44d948f77809213cb4977_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:39,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55e3335733949da816ec44c07bf8e1d, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:39,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55e3335733949da816ec44c07bf8e1d is 175, key is test_row_0/A:col10/1731296438876/Put/seqid=0 2024-11-11T03:40:39,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742045_1221 (size=39949) 2024-11-11T03:40:39,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55e3335733949da816ec44c07bf8e1d 2024-11-11T03:40:39,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/2c9d6bf001a14cb6bf268ea544bf8485 is 50, key is test_row_0/B:col10/1731296438876/Put/seqid=0 2024-11-11T03:40:39,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742046_1222 (size=12301) 2024-11-11T03:40:39,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/2c9d6bf001a14cb6bf268ea544bf8485 2024-11-11T03:40:39,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/8a350b2c34804ea1afa7c654a5134084 is 50, key is test_row_0/C:col10/1731296438876/Put/seqid=0 2024-11-11T03:40:39,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742047_1223 (size=12301) 2024-11-11T03:40:39,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/8a350b2c34804ea1afa7c654a5134084 2024-11-11T03:40:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/f55e3335733949da816ec44c07bf8e1d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55e3335733949da816ec44c07bf8e1d 2024-11-11T03:40:39,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55e3335733949da816ec44c07bf8e1d, entries=200, sequenceid=329, filesize=39.0 K 2024-11-11T03:40:39,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/2c9d6bf001a14cb6bf268ea544bf8485 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2c9d6bf001a14cb6bf268ea544bf8485 2024-11-11T03:40:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2c9d6bf001a14cb6bf268ea544bf8485, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T03:40:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/8a350b2c34804ea1afa7c654a5134084 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8a350b2c34804ea1afa7c654a5134084 2024-11-11T03:40:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8a350b2c34804ea1afa7c654a5134084, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T03:40:39,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 887247b757faef0e5f5ee791239fb929 in 101ms, sequenceid=329, compaction requested=true 2024-11-11T03:40:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:39,101 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:39,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:39,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:39,101 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:39,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:39,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:39,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:39,102 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:39,102 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:39,102 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:39,102 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:39,102 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,102 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,102 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/eceed955dab84c4db55480bd2fc47c89, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f8112cd31cf74db9a80665a57dd0a7d0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55e3335733949da816ec44c07bf8e1d] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=92.2 K 2024-11-11T03:40:39,102 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/b4d3b7cbffe543ff840cdec1c3fc372c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4294fbe8d0cc46839400de947158f08b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2c9d6bf001a14cb6bf268ea544bf8485] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=34.3 K 2024-11-11T03:40:39,102 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,102 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/eceed955dab84c4db55480bd2fc47c89, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f8112cd31cf74db9a80665a57dd0a7d0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55e3335733949da816ec44c07bf8e1d] 2024-11-11T03:40:39,103 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d3b7cbffe543ff840cdec1c3fc372c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731296434445 2024-11-11T03:40:39,103 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting eceed955dab84c4db55480bd2fc47c89, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731296434445 2024-11-11T03:40:39,104 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4294fbe8d0cc46839400de947158f08b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731296436635 2024-11-11T03:40:39,104 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8112cd31cf74db9a80665a57dd0a7d0, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731296436635 2024-11-11T03:40:39,104 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9d6bf001a14cb6bf268ea544bf8485, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296438876 2024-11-11T03:40:39,104 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55e3335733949da816ec44c07bf8e1d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296438876 2024-11-11T03:40:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,112 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,113 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,114 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7305666883bf4e73b4d8d9c730f728cf is 50, key is test_row_0/B:col10/1731296438876/Put/seqid=0 2024-11-11T03:40:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,115 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111163b1ecbb13e2462a881db09ce50d9914_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,117 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111163b1ecbb13e2462a881db09ce50d9914_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:39,117 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111163b1ecbb13e2462a881db09ce50d9914_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742048_1224 (size=13085) 2024-11-11T03:40:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:39,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:40:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:39,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:39,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:39,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:39,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:39,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742049_1225 (size=4469) 2024-11-11T03:40:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,136 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7305666883bf4e73b4d8d9c730f728cf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7305666883bf4e73b4d8d9c730f728cf 2024-11-11T03:40:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,139 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#186 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,140 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/8009998ea04449fe8a69c650026dd26a is 175, key is test_row_0/A:col10/1731296438876/Put/seqid=0 2024-11-11T03:40:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,145 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into 7305666883bf4e73b4d8d9c730f728cf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,145 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:39,145 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=13, startTime=1731296439101; duration=0sec 2024-11-11T03:40:39,145 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:39,145 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:39,146 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,147 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,147 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:39,147 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,148 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6805ba34788a443987aa1531eb140fc7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/72a3a570035f4352a318597e3adf9d27, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8a350b2c34804ea1afa7c654a5134084] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=34.3 K 2024-11-11T03:40:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,148 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6805ba34788a443987aa1531eb140fc7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731296434445 2024-11-11T03:40:39,149 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a3a570035f4352a318597e3adf9d27, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731296436635 2024-11-11T03:40:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,150 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a350b2c34804ea1afa7c654a5134084, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296438876 2024-11-11T03:40:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111162db4e778ae4498286cd01abc9a68ab3_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296439005/Put/seqid=0 2024-11-11T03:40:39,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742050_1226 (size=32039) 2024-11-11T03:40:39,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731296439155, max timestamp=1731296439170Num uniques 859; 2024-11-11T03:40:39,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731296439155, max timestamp=1731296439170Num uniques 859; 2024-11-11T03:40:39,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T03:40:39,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T03:40:39,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731296439155, max timestamp=1731296439170Num uniques 859; 2024-11-11T03:40:39,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T03:40:39,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742051_1227 (size=14994) 2024-11-11T03:40:39,179 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:39,180 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/71b8bb7311e14275bf6bc2827e64f627 is 50, key is test_row_0/C:col10/1731296438876/Put/seqid=0 2024-11-11T03:40:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742052_1228 (size=13085) 2024-11-11T03:40:39,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296499192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296499193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296499194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296499200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296499201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296499312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296499312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296499312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296499312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296499312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T03:40:39,462 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:39,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T03:40:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T03:40:39,464 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:39,464 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:39,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296499522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296499522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296499522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296499522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296499522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T03:40:39,574 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/8009998ea04449fe8a69c650026dd26a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8009998ea04449fe8a69c650026dd26a 2024-11-11T03:40:39,577 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:39,579 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into 8009998ea04449fe8a69c650026dd26a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:39,579 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:39,579 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=13, startTime=1731296439100; duration=0sec 2024-11-11T03:40:39,579 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:39,579 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:39,580 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111162db4e778ae4498286cd01abc9a68ab3_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111162db4e778ae4498286cd01abc9a68ab3_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:39,581 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/465590f9126649a696cd4d17925bbdb7, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:39,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/465590f9126649a696cd4d17925bbdb7 is 175, key is test_row_0/A:col10/1731296439005/Put/seqid=0 2024-11-11T03:40:39,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742053_1229 (size=39949) 2024-11-11T03:40:39,600 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/71b8bb7311e14275bf6bc2827e64f627 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/71b8bb7311e14275bf6bc2827e64f627 2024-11-11T03:40:39,606 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into 71b8bb7311e14275bf6bc2827e64f627(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:39,606 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:39,606 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=13, startTime=1731296439101; duration=0sec 2024-11-11T03:40:39,606 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:39,606 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:39,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T03:40:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T03:40:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T03:40:39,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296499831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296499831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296499831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296499832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296499832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:39,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T03:40:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:39,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:39,989 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/465590f9126649a696cd4d17925bbdb7 2024-11-11T03:40:39,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/771fdd6ad5b54c1b9a9d5744605a7bdb is 50, key is test_row_0/B:col10/1731296439005/Put/seqid=0 2024-11-11T03:40:40,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742054_1230 (size=12301) 2024-11-11T03:40:40,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/771fdd6ad5b54c1b9a9d5744605a7bdb 2024-11-11T03:40:40,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/791d02d07c0341d688c71c7c495b69f2 is 50, key is test_row_0/C:col10/1731296439005/Put/seqid=0 2024-11-11T03:40:40,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742055_1231 (size=12301) 2024-11-11T03:40:40,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/791d02d07c0341d688c71c7c495b69f2 2024-11-11T03:40:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/465590f9126649a696cd4d17925bbdb7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/465590f9126649a696cd4d17925bbdb7 2024-11-11T03:40:40,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/465590f9126649a696cd4d17925bbdb7, entries=200, sequenceid=339, filesize=39.0 K 2024-11-11T03:40:40,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/771fdd6ad5b54c1b9a9d5744605a7bdb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/771fdd6ad5b54c1b9a9d5744605a7bdb 2024-11-11T03:40:40,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/771fdd6ad5b54c1b9a9d5744605a7bdb, entries=150, sequenceid=339, filesize=12.0 K 2024-11-11T03:40:40,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/791d02d07c0341d688c71c7c495b69f2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/791d02d07c0341d688c71c7c495b69f2 2024-11-11T03:40:40,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/791d02d07c0341d688c71c7c495b69f2, entries=150, sequenceid=339, filesize=12.0 K 2024-11-11T03:40:40,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 887247b757faef0e5f5ee791239fb929 in 908ms, sequenceid=339, compaction requested=false 2024-11-11T03:40:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:40,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T03:40:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:40,075 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-11T03:40:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:40,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111706d55abba3f4a3bac151111545465c5_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296439199/Put/seqid=0 2024-11-11T03:40:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T03:40:40,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742056_1232 (size=12454) 2024-11-11T03:40:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:40,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:40,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296500359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296500360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296500361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296500362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296500362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296500471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296500472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296500472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296500472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296500472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:40,498 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111706d55abba3f4a3bac151111545465c5_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111706d55abba3f4a3bac151111545465c5_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/1a6a8839af184585928a76de802f5863, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/1a6a8839af184585928a76de802f5863 is 175, key is test_row_0/A:col10/1731296439199/Put/seqid=0 2024-11-11T03:40:40,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742057_1233 (size=31255) 2024-11-11T03:40:40,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/1a6a8839af184585928a76de802f5863 2024-11-11T03:40:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/2a868c78d4db451d8aa100d9963bc998 is 50, key is test_row_0/B:col10/1731296439199/Put/seqid=0 2024-11-11T03:40:40,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742058_1234 (size=12301) 2024-11-11T03:40:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T03:40:40,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296500682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296500682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296500682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296500682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296500682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,918 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/2a868c78d4db451d8aa100d9963bc998 2024-11-11T03:40:40,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/1475e388417b4d019122e448f3b5d0d8 is 50, key is test_row_0/C:col10/1731296439199/Put/seqid=0 2024-11-11T03:40:40,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742059_1235 (size=12301) 2024-11-11T03:40:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296500991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296500991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296500992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296500992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296500992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,329 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/1475e388417b4d019122e448f3b5d0d8 2024-11-11T03:40:41,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/1a6a8839af184585928a76de802f5863 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/1a6a8839af184585928a76de802f5863 2024-11-11T03:40:41,336 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/1a6a8839af184585928a76de802f5863, entries=150, sequenceid=376, filesize=30.5 K 2024-11-11T03:40:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/2a868c78d4db451d8aa100d9963bc998 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2a868c78d4db451d8aa100d9963bc998 2024-11-11T03:40:41,341 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2a868c78d4db451d8aa100d9963bc998, entries=150, sequenceid=376, filesize=12.0 K 2024-11-11T03:40:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/1475e388417b4d019122e448f3b5d0d8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1475e388417b4d019122e448f3b5d0d8 2024-11-11T03:40:41,345 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1475e388417b4d019122e448f3b5d0d8, entries=150, sequenceid=376, filesize=12.0 K 2024-11-11T03:40:41,346 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=60.38 KB/61830 for 887247b757faef0e5f5ee791239fb929 in 1271ms, sequenceid=376, compaction requested=true 2024-11-11T03:40:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T03:40:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T03:40:41,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T03:40:41,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-11T03:40:41,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8880 sec 2024-11-11T03:40:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:41,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T03:40:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:41,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111af81071734de41fd9dd99e83104857fb_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296441503/Put/seqid=0 2024-11-11T03:40:41,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742060_1236 (size=17534) 2024-11-11T03:40:41,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296501521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296501522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296501527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296501528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296501530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T03:40:41,612 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:41,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-11T03:40:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T03:40:41,614 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:41,615 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:41,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:41,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296501632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296501641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296501641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296501642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296501642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T03:40:41,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:41,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296501841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296501852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296501852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296501852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296501852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:41,917 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:41,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:41,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:41,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111af81071734de41fd9dd99e83104857fb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111af81071734de41fd9dd99e83104857fb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:41,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/9d9c6fe5cefb47588849ba90a20654cd, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:41,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/9d9c6fe5cefb47588849ba90a20654cd is 175, key is test_row_0/A:col10/1731296441503/Put/seqid=0 2024-11-11T03:40:41,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742061_1237 (size=48639) 2024-11-11T03:40:41,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=390, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/9d9c6fe5cefb47588849ba90a20654cd 2024-11-11T03:40:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T03:40:41,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4dae56c851844b4883de2d8b5e6fa66b is 50, key is test_row_0/B:col10/1731296441503/Put/seqid=0 2024-11-11T03:40:41,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742062_1238 (size=12301) 2024-11-11T03:40:42,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:42,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:42,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,079 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-11T03:40:42,079 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:42,079 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,079 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:42,080 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-11-11T03:40:42,080 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:42,080 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,080 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:42,082 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-11-11T03:40:42,082 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:42,082 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,082 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:42,083 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-11T03:40:42,083 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:42,083 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,083 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:42,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296502152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296502162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296502162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296502163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296502163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:42,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:42,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T03:40:42,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4dae56c851844b4883de2d8b5e6fa66b 2024-11-11T03:40:42,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/76e85f0752664040b38548c563d27bb4 is 50, key is test_row_0/C:col10/1731296441503/Put/seqid=0 2024-11-11T03:40:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742063_1239 (size=12301) 2024-11-11T03:40:42,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:42,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:42,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,621 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:40:42,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35208 deadline: 1731296502662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35178 deadline: 1731296502672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35218 deadline: 1731296502673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35216 deadline: 1731296502673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35224 deadline: 1731296502673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:40:42,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T03:40:42,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/76e85f0752664040b38548c563d27bb4 2024-11-11T03:40:42,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/9d9c6fe5cefb47588849ba90a20654cd as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/9d9c6fe5cefb47588849ba90a20654cd 2024-11-11T03:40:42,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/9d9c6fe5cefb47588849ba90a20654cd, entries=250, sequenceid=390, filesize=47.5 K 2024-11-11T03:40:42,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/4dae56c851844b4883de2d8b5e6fa66b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4dae56c851844b4883de2d8b5e6fa66b 2024-11-11T03:40:42,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4dae56c851844b4883de2d8b5e6fa66b, entries=150, sequenceid=390, filesize=12.0 K 2024-11-11T03:40:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/76e85f0752664040b38548c563d27bb4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/76e85f0752664040b38548c563d27bb4 2024-11-11T03:40:42,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/76e85f0752664040b38548c563d27bb4, entries=150, sequenceid=390, filesize=12.0 K 2024-11-11T03:40:42,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 887247b757faef0e5f5ee791239fb929 in 1287ms, sequenceid=390, compaction requested=true 2024-11-11T03:40:42,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:42,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:42,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:42,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:42,790 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:42,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:42,790 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:42,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 887247b757faef0e5f5ee791239fb929:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:42,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:42,791 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:42,791 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151882 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:42,791 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/B is initiating minor compaction (all files) 2024-11-11T03:40:42,791 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/A is initiating minor compaction (all files) 2024-11-11T03:40:42,791 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/B in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,791 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/A in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,791 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8009998ea04449fe8a69c650026dd26a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/465590f9126649a696cd4d17925bbdb7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/1a6a8839af184585928a76de802f5863, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/9d9c6fe5cefb47588849ba90a20654cd] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=148.3 K 2024-11-11T03:40:42,791 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7305666883bf4e73b4d8d9c730f728cf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/771fdd6ad5b54c1b9a9d5744605a7bdb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2a868c78d4db451d8aa100d9963bc998, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4dae56c851844b4883de2d8b5e6fa66b] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=48.8 K 2024-11-11T03:40:42,792 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,792 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8009998ea04449fe8a69c650026dd26a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/465590f9126649a696cd4d17925bbdb7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/1a6a8839af184585928a76de802f5863, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/9d9c6fe5cefb47588849ba90a20654cd] 2024-11-11T03:40:42,792 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 7305666883bf4e73b4d8d9c730f728cf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296438876 2024-11-11T03:40:42,792 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8009998ea04449fe8a69c650026dd26a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296438876 2024-11-11T03:40:42,792 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 771fdd6ad5b54c1b9a9d5744605a7bdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731296439005 2024-11-11T03:40:42,792 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 465590f9126649a696cd4d17925bbdb7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731296439005 2024-11-11T03:40:42,792 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a6a8839af184585928a76de802f5863, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731296439191 2024-11-11T03:40:42,793 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a868c78d4db451d8aa100d9963bc998, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731296439191 2024-11-11T03:40:42,793 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9c6fe5cefb47588849ba90a20654cd, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731296440359 2024-11-11T03:40:42,793 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dae56c851844b4883de2d8b5e6fa66b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731296440359 2024-11-11T03:40:42,801 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#B#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:42,802 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/9ae301ba30194e8baab4c39ac3f0c2bc is 50, key is test_row_0/B:col10/1731296441503/Put/seqid=0 2024-11-11T03:40:42,803 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742064_1240 (size=13221) 2024-11-11T03:40:42,807 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111578abf42abea479aaa99e2cc033f30ce_887247b757faef0e5f5ee791239fb929 store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:42,828 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111578abf42abea479aaa99e2cc033f30ce_887247b757faef0e5f5ee791239fb929, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:42,828 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111578abf42abea479aaa99e2cc033f30ce_887247b757faef0e5f5ee791239fb929 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:42,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742065_1241 (size=4469) 2024-11-11T03:40:42,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T03:40:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:42,854 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T03:40:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:42,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:42,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111018825e4a824c50b94bc2b7062b187c_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296441520/Put/seqid=0 2024-11-11T03:40:42,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742066_1242 (size=12454) 2024-11-11T03:40:43,213 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/9ae301ba30194e8baab4c39ac3f0c2bc as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9ae301ba30194e8baab4c39ac3f0c2bc 2024-11-11T03:40:43,221 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/B of 887247b757faef0e5f5ee791239fb929 into 9ae301ba30194e8baab4c39ac3f0c2bc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:43,221 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:43,221 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/B, priority=12, startTime=1731296442790; duration=0sec 2024-11-11T03:40:43,221 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:43,221 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:B 2024-11-11T03:40:43,221 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:40:43,223 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:40:43,223 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 887247b757faef0e5f5ee791239fb929/C is initiating minor compaction (all files) 2024-11-11T03:40:43,223 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 887247b757faef0e5f5ee791239fb929/C in TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:43,223 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/71b8bb7311e14275bf6bc2827e64f627, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/791d02d07c0341d688c71c7c495b69f2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1475e388417b4d019122e448f3b5d0d8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/76e85f0752664040b38548c563d27bb4] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp, totalSize=48.8 K 2024-11-11T03:40:43,224 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b8bb7311e14275bf6bc2827e64f627, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296438876 2024-11-11T03:40:43,224 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 791d02d07c0341d688c71c7c495b69f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731296439005 2024-11-11T03:40:43,224 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 1475e388417b4d019122e448f3b5d0d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731296439191 2024-11-11T03:40:43,224 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 76e85f0752664040b38548c563d27bb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731296440359 2024-11-11T03:40:43,232 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#C#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:43,233 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/873c912c5b5f41bc894a490f1c86ac62 is 50, key is test_row_0/C:col10/1731296441503/Put/seqid=0 2024-11-11T03:40:43,233 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 887247b757faef0e5f5ee791239fb929#A#compaction#198 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:43,234 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0fe39168105a429da6d89beafbb66c4a is 175, key is test_row_0/A:col10/1731296441503/Put/seqid=0 2024-11-11T03:40:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742067_1243 (size=13221) 2024-11-11T03:40:43,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742068_1244 (size=32175) 2024-11-11T03:40:43,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:43,275 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111018825e4a824c50b94bc2b7062b187c_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111018825e4a824c50b94bc2b7062b187c_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:43,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/73d351a75f2b45a19d28f09ff02fa396, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:43,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/73d351a75f2b45a19d28f09ff02fa396 is 175, key is test_row_0/A:col10/1731296441520/Put/seqid=0 2024-11-11T03:40:43,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742069_1245 (size=31255) 2024-11-11T03:40:43,646 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/873c912c5b5f41bc894a490f1c86ac62 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/873c912c5b5f41bc894a490f1c86ac62 2024-11-11T03:40:43,646 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/0fe39168105a429da6d89beafbb66c4a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0fe39168105a429da6d89beafbb66c4a 2024-11-11T03:40:43,653 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/A of 887247b757faef0e5f5ee791239fb929 into 0fe39168105a429da6d89beafbb66c4a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:43,653 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 887247b757faef0e5f5ee791239fb929/C of 887247b757faef0e5f5ee791239fb929 into 873c912c5b5f41bc894a490f1c86ac62(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:43,653 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:43,653 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:43,654 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/A, priority=12, startTime=1731296442790; duration=0sec 2024-11-11T03:40:43,654 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., storeName=887247b757faef0e5f5ee791239fb929/C, priority=12, startTime=1731296442790; duration=0sec 2024-11-11T03:40:43,654 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:43,654 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:A 2024-11-11T03:40:43,654 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:43,654 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 887247b757faef0e5f5ee791239fb929:C 2024-11-11T03:40:43,685 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/73d351a75f2b45a19d28f09ff02fa396 2024-11-11T03:40:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. as already flushing 2024-11-11T03:40:43,687 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-11T03:40:43,687 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:43,687 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,687 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-11T03:40:43,687 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-11T03:40:43,687 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,687 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:43,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:43,687 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:43,687 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,687 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,687 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,687 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:43,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:43,693 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-11T03:40:43,693 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:43,693 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,693 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-11T03:40:43,693 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,693 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:40:43,694 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:43,694 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:43,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:43,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7ee6124d845d476dbe1529ff7d283a95 is 50, key is test_row_0/B:col10/1731296441520/Put/seqid=0 2024-11-11T03:40:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742070_1246 (size=12301) 2024-11-11T03:40:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T03:40:44,102 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7ee6124d845d476dbe1529ff7d283a95 2024-11-11T03:40:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/3ef15184702c4952b1431478269cc83d is 50, key is test_row_0/C:col10/1731296441520/Put/seqid=0 2024-11-11T03:40:44,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742071_1247 (size=12301) 2024-11-11T03:40:44,524 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/3ef15184702c4952b1431478269cc83d 2024-11-11T03:40:44,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/73d351a75f2b45a19d28f09ff02fa396 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/73d351a75f2b45a19d28f09ff02fa396 2024-11-11T03:40:44,540 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/73d351a75f2b45a19d28f09ff02fa396, entries=150, sequenceid=408, filesize=30.5 K 2024-11-11T03:40:44,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/7ee6124d845d476dbe1529ff7d283a95 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7ee6124d845d476dbe1529ff7d283a95 2024-11-11T03:40:44,546 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7ee6124d845d476dbe1529ff7d283a95, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T03:40:44,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/3ef15184702c4952b1431478269cc83d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/3ef15184702c4952b1431478269cc83d 2024-11-11T03:40:44,552 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/3ef15184702c4952b1431478269cc83d, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T03:40:44,553 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=33.54 KB/34350 for 887247b757faef0e5f5ee791239fb929 in 1699ms, sequenceid=408, compaction requested=false 2024-11-11T03:40:44,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 887247b757faef0e5f5ee791239fb929: 2024-11-11T03:40:44,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:44,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-11T03:40:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-11T03:40:44,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T03:40:44,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9400 sec 2024-11-11T03:40:44,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.9440 sec 2024-11-11T03:40:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T03:40:45,773 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3019 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3055 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1309 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3927 rows 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1307 2024-11-11T03:40:45,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3918 rows 2024-11-11T03:40:45,774 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:40:45,774 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:40:45,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:45,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:45,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:40:45,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T03:40:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T03:40:45,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296445783"}]},"ts":"1731296445783"} 2024-11-11T03:40:45,785 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T03:40:45,785 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T03:40:45,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:40:45,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, UNASSIGN}] 2024-11-11T03:40:45,789 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, UNASSIGN 2024-11-11T03:40:45,790 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:45,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, UNASSIGN because future has completed 2024-11-11T03:40:45,792 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:40:45,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:40:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T03:40:45,947 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:45,947 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:40:45,947 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 887247b757faef0e5f5ee791239fb929, disabling compactions & flushes 2024-11-11T03:40:45,947 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:45,947 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:45,948 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. after waiting 0 ms 2024-11-11T03:40:45,948 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:45,948 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 887247b757faef0e5f5ee791239fb929 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T03:40:45,949 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=A 2024-11-11T03:40:45,949 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:45,949 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=B 2024-11-11T03:40:45,949 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:45,949 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 887247b757faef0e5f5ee791239fb929, store=C 2024-11-11T03:40:45,949 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:45,959 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c508b69604ec4dc394cbcabd42740c2a_887247b757faef0e5f5ee791239fb929 is 50, key is test_row_0/A:col10/1731296443692/Put/seqid=0 2024-11-11T03:40:45,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742072_1248 (size=12454) 2024-11-11T03:40:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T03:40:46,364 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:46,372 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c508b69604ec4dc394cbcabd42740c2a_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c508b69604ec4dc394cbcabd42740c2a_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:46,374 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/6ee3129e0d6e4367b9c186e6c6d6cbaf, store: [table=TestAcidGuarantees family=A region=887247b757faef0e5f5ee791239fb929] 2024-11-11T03:40:46,374 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/6ee3129e0d6e4367b9c186e6c6d6cbaf is 175, key is test_row_0/A:col10/1731296443692/Put/seqid=0 2024-11-11T03:40:46,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742073_1249 (size=31255) 2024-11-11T03:40:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T03:40:46,782 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/6ee3129e0d6e4367b9c186e6c6d6cbaf 2024-11-11T03:40:46,796 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/20f4aacc4f464799bed13cb3ae512be4 is 50, key is test_row_0/B:col10/1731296443692/Put/seqid=0 2024-11-11T03:40:46,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742074_1250 (size=12301) 2024-11-11T03:40:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T03:40:47,203 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/20f4aacc4f464799bed13cb3ae512be4 2024-11-11T03:40:47,252 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/c139c1b27d094faf8a856fef597d17b1 is 50, key is test_row_0/C:col10/1731296443692/Put/seqid=0 2024-11-11T03:40:47,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742075_1251 (size=12301) 2024-11-11T03:40:47,657 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/c139c1b27d094faf8a856fef597d17b1 2024-11-11T03:40:47,664 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/A/6ee3129e0d6e4367b9c186e6c6d6cbaf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/6ee3129e0d6e4367b9c186e6c6d6cbaf 2024-11-11T03:40:47,669 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/6ee3129e0d6e4367b9c186e6c6d6cbaf, entries=150, sequenceid=419, filesize=30.5 K 2024-11-11T03:40:47,670 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/B/20f4aacc4f464799bed13cb3ae512be4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/20f4aacc4f464799bed13cb3ae512be4 2024-11-11T03:40:47,675 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/20f4aacc4f464799bed13cb3ae512be4, entries=150, sequenceid=419, filesize=12.0 K 2024-11-11T03:40:47,676 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/.tmp/C/c139c1b27d094faf8a856fef597d17b1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/c139c1b27d094faf8a856fef597d17b1 2024-11-11T03:40:47,681 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/c139c1b27d094faf8a856fef597d17b1, entries=150, sequenceid=419, filesize=12.0 K 2024-11-11T03:40:47,681 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 887247b757faef0e5f5ee791239fb929 in 1733ms, sequenceid=419, compaction requested=true 2024-11-11T03:40:47,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/d3609efe2eaa42409522e84ef5787d28, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55130c3d75d484b97bf429f4d76f0fb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/cd12e64ff6e54cfa90c4cfaed2489350, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0f1525138914495a821173d29f676273, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/b5c41cc0c1ff4c2cbee938fc265cf21b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/dd86286032824d098431d0c2f021d31b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/382a889e8c91480ab4fca7f07b850cb0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8464e4493f774d6d91b01436110bd54d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/322a1dc9e52a488ea777c811281d2121, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0b5667bca59c4060932fab042f356419, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e60a577d0e9d41c98673dfd645a81c62, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/59cda0c847424f3dacd7e7ddc8f886e0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/3d78ca7f34b84626855760754b305b5c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/65ac8dbd14ba4bdcb20214d4ee277477, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/fa57d22b9fed43e7919ad627a960a3c8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/58550c674b124f89ac3213eb726c10f1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e6f4a28f5c4741ee89a903d80b5654fb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/ad76ea550c5d478e8d06777e1b5fcc03, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/439c3a1445f146e9ba083894e0584169, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/31638a8fb8b0462a8d12a9e10497e90a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/eceed955dab84c4db55480bd2fc47c89, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f8112cd31cf74db9a80665a57dd0a7d0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55e3335733949da816ec44c07bf8e1d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8009998ea04449fe8a69c650026dd26a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/465590f9126649a696cd4d17925bbdb7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/1a6a8839af184585928a76de802f5863, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/9d9c6fe5cefb47588849ba90a20654cd] to archive 2024-11-11T03:40:47,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:40:47,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/d3609efe2eaa42409522e84ef5787d28 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/d3609efe2eaa42409522e84ef5787d28 2024-11-11T03:40:47,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55130c3d75d484b97bf429f4d76f0fb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55130c3d75d484b97bf429f4d76f0fb 2024-11-11T03:40:47,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/cd12e64ff6e54cfa90c4cfaed2489350 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/cd12e64ff6e54cfa90c4cfaed2489350 2024-11-11T03:40:47,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0f1525138914495a821173d29f676273 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0f1525138914495a821173d29f676273 2024-11-11T03:40:47,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/b5c41cc0c1ff4c2cbee938fc265cf21b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/b5c41cc0c1ff4c2cbee938fc265cf21b 2024-11-11T03:40:47,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/dd86286032824d098431d0c2f021d31b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/dd86286032824d098431d0c2f021d31b 2024-11-11T03:40:47,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/382a889e8c91480ab4fca7f07b850cb0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/382a889e8c91480ab4fca7f07b850cb0 2024-11-11T03:40:47,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8464e4493f774d6d91b01436110bd54d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8464e4493f774d6d91b01436110bd54d 2024-11-11T03:40:47,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/322a1dc9e52a488ea777c811281d2121 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/322a1dc9e52a488ea777c811281d2121 2024-11-11T03:40:47,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0b5667bca59c4060932fab042f356419 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0b5667bca59c4060932fab042f356419 2024-11-11T03:40:47,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e60a577d0e9d41c98673dfd645a81c62 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e60a577d0e9d41c98673dfd645a81c62 2024-11-11T03:40:47,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/59cda0c847424f3dacd7e7ddc8f886e0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/59cda0c847424f3dacd7e7ddc8f886e0 2024-11-11T03:40:47,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/3d78ca7f34b84626855760754b305b5c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/3d78ca7f34b84626855760754b305b5c 2024-11-11T03:40:47,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/65ac8dbd14ba4bdcb20214d4ee277477 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/65ac8dbd14ba4bdcb20214d4ee277477 2024-11-11T03:40:47,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/fa57d22b9fed43e7919ad627a960a3c8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/fa57d22b9fed43e7919ad627a960a3c8 2024-11-11T03:40:47,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/58550c674b124f89ac3213eb726c10f1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/58550c674b124f89ac3213eb726c10f1 2024-11-11T03:40:47,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e6f4a28f5c4741ee89a903d80b5654fb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/e6f4a28f5c4741ee89a903d80b5654fb 2024-11-11T03:40:47,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/ad76ea550c5d478e8d06777e1b5fcc03 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/ad76ea550c5d478e8d06777e1b5fcc03 2024-11-11T03:40:47,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/439c3a1445f146e9ba083894e0584169 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/439c3a1445f146e9ba083894e0584169 2024-11-11T03:40:47,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/31638a8fb8b0462a8d12a9e10497e90a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/31638a8fb8b0462a8d12a9e10497e90a 2024-11-11T03:40:47,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/eceed955dab84c4db55480bd2fc47c89 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/eceed955dab84c4db55480bd2fc47c89 2024-11-11T03:40:47,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f8112cd31cf74db9a80665a57dd0a7d0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f8112cd31cf74db9a80665a57dd0a7d0 2024-11-11T03:40:47,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55e3335733949da816ec44c07bf8e1d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/f55e3335733949da816ec44c07bf8e1d 2024-11-11T03:40:47,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8009998ea04449fe8a69c650026dd26a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/8009998ea04449fe8a69c650026dd26a 2024-11-11T03:40:47,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/465590f9126649a696cd4d17925bbdb7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/465590f9126649a696cd4d17925bbdb7 2024-11-11T03:40:47,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/1a6a8839af184585928a76de802f5863 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/1a6a8839af184585928a76de802f5863 2024-11-11T03:40:47,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/9d9c6fe5cefb47588849ba90a20654cd to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/9d9c6fe5cefb47588849ba90a20654cd 2024-11-11T03:40:47,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4f625d685b5b4e0eb109bc3b04a5e77b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3d8c5b299ec649deba1c1876991cb2fd, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/557e28fa7d3448fdaacbe1f462f4ca0b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/f4f75e51b217476e839be38bec0501e7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/cb1fbf8a88194bfca272747afe690ec2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3be73bcffe53489882171a10e741a8cc, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d535b2f180bb4f8db4a91841e76bbe08, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d52af7b428a94e31b40c0cca5bf6bd64, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/0a6c734b6a93497182bb9c3b829671c4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/921510fd85b643b68d7cfe0b3563f9a5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/c145fddc3a34460baadf403b62ba12b4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d3715962cf1c4a7bb3844fd3a23470fe, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/56d5e9f0d4bb41aabac401af2bb43cda, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9f2e028fa6374ab88e062ba8f2d32843, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/038c3b1d9e9b4e76b2a309c44b724f1e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/41a1a24edaaf47f9bb61f38425ad889f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/937e6c11893448c0b8a2185dc53dc42e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7fa7e2028ea94f3c879d3ef7f2378db9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/44c8ce1b6a1e4c30af243c7342c496c0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/b4d3b7cbffe543ff840cdec1c3fc372c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3208d721bcd1417e9b8c5a2f112813e1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4294fbe8d0cc46839400de947158f08b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7305666883bf4e73b4d8d9c730f728cf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2c9d6bf001a14cb6bf268ea544bf8485, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/771fdd6ad5b54c1b9a9d5744605a7bdb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2a868c78d4db451d8aa100d9963bc998, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4dae56c851844b4883de2d8b5e6fa66b] to archive 2024-11-11T03:40:47,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:40:47,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4f625d685b5b4e0eb109bc3b04a5e77b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4f625d685b5b4e0eb109bc3b04a5e77b 2024-11-11T03:40:47,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3d8c5b299ec649deba1c1876991cb2fd to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3d8c5b299ec649deba1c1876991cb2fd 2024-11-11T03:40:47,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/557e28fa7d3448fdaacbe1f462f4ca0b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/557e28fa7d3448fdaacbe1f462f4ca0b 2024-11-11T03:40:47,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/f4f75e51b217476e839be38bec0501e7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/f4f75e51b217476e839be38bec0501e7 2024-11-11T03:40:47,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/cb1fbf8a88194bfca272747afe690ec2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/cb1fbf8a88194bfca272747afe690ec2 2024-11-11T03:40:47,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3be73bcffe53489882171a10e741a8cc to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3be73bcffe53489882171a10e741a8cc 2024-11-11T03:40:47,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d535b2f180bb4f8db4a91841e76bbe08 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d535b2f180bb4f8db4a91841e76bbe08 2024-11-11T03:40:47,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d52af7b428a94e31b40c0cca5bf6bd64 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d52af7b428a94e31b40c0cca5bf6bd64 2024-11-11T03:40:47,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/0a6c734b6a93497182bb9c3b829671c4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/0a6c734b6a93497182bb9c3b829671c4 2024-11-11T03:40:47,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/921510fd85b643b68d7cfe0b3563f9a5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/921510fd85b643b68d7cfe0b3563f9a5 2024-11-11T03:40:47,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/c145fddc3a34460baadf403b62ba12b4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/c145fddc3a34460baadf403b62ba12b4 2024-11-11T03:40:47,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d3715962cf1c4a7bb3844fd3a23470fe to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/d3715962cf1c4a7bb3844fd3a23470fe 2024-11-11T03:40:47,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/56d5e9f0d4bb41aabac401af2bb43cda to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/56d5e9f0d4bb41aabac401af2bb43cda 2024-11-11T03:40:47,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9f2e028fa6374ab88e062ba8f2d32843 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9f2e028fa6374ab88e062ba8f2d32843 2024-11-11T03:40:47,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/038c3b1d9e9b4e76b2a309c44b724f1e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/038c3b1d9e9b4e76b2a309c44b724f1e 2024-11-11T03:40:47,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/41a1a24edaaf47f9bb61f38425ad889f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/41a1a24edaaf47f9bb61f38425ad889f 2024-11-11T03:40:47,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/937e6c11893448c0b8a2185dc53dc42e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/937e6c11893448c0b8a2185dc53dc42e 2024-11-11T03:40:47,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7fa7e2028ea94f3c879d3ef7f2378db9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7fa7e2028ea94f3c879d3ef7f2378db9 2024-11-11T03:40:47,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/44c8ce1b6a1e4c30af243c7342c496c0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/44c8ce1b6a1e4c30af243c7342c496c0 2024-11-11T03:40:47,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/b4d3b7cbffe543ff840cdec1c3fc372c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/b4d3b7cbffe543ff840cdec1c3fc372c 2024-11-11T03:40:47,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3208d721bcd1417e9b8c5a2f112813e1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/3208d721bcd1417e9b8c5a2f112813e1 2024-11-11T03:40:47,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4294fbe8d0cc46839400de947158f08b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4294fbe8d0cc46839400de947158f08b 2024-11-11T03:40:47,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7305666883bf4e73b4d8d9c730f728cf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7305666883bf4e73b4d8d9c730f728cf 2024-11-11T03:40:47,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2c9d6bf001a14cb6bf268ea544bf8485 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2c9d6bf001a14cb6bf268ea544bf8485 2024-11-11T03:40:47,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/771fdd6ad5b54c1b9a9d5744605a7bdb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/771fdd6ad5b54c1b9a9d5744605a7bdb 2024-11-11T03:40:47,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2a868c78d4db451d8aa100d9963bc998 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/2a868c78d4db451d8aa100d9963bc998 2024-11-11T03:40:47,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4dae56c851844b4883de2d8b5e6fa66b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/4dae56c851844b4883de2d8b5e6fa66b 2024-11-11T03:40:47,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/2a81ccbf5e894e26b6e4055532c58688, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/653eee7d9f7747cfb892f4a85993e94d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/553ea3f2270942959e78194b90559c8f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6e126d23c35b47d885b9dc8760d643c8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/f7625d34419f4670a86eaf482f99a7f9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/a17a7de85fe64585801da8604f385064, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8c0339df2f6b4799a1bdb36338b749c4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cbf53d5eabc540a5bb30dcdeff546890, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/e3d2a7b06e104577a82d997fab2392ed, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cfcca77560b5440e8af1251214edd132, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/4789111e644846caad1f0fac9af84caa, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b7e3f71238164ada92389e8b704adec1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/329437a494194265869181fda9347fcc, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b86b09e893de4b048d31d9811bb5b883, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/fa8b4008ac3544588983756f1d6496ce, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1abd50d4956643fcba6a72d321ff8d97, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/859e10e3816c4456843f0e74c4a07100, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/0875c7740568481ab8bfaf42b6a646ee, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/69dec9fe52dd41daa8987e8071413620, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6805ba34788a443987aa1531eb140fc7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/05a678a1618b41afaec2bba3edc57fd9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/72a3a570035f4352a318597e3adf9d27, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/71b8bb7311e14275bf6bc2827e64f627, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8a350b2c34804ea1afa7c654a5134084, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/791d02d07c0341d688c71c7c495b69f2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1475e388417b4d019122e448f3b5d0d8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/76e85f0752664040b38548c563d27bb4] to archive 2024-11-11T03:40:47,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:40:47,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/2a81ccbf5e894e26b6e4055532c58688 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/2a81ccbf5e894e26b6e4055532c58688 2024-11-11T03:40:47,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/653eee7d9f7747cfb892f4a85993e94d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/653eee7d9f7747cfb892f4a85993e94d 2024-11-11T03:40:47,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/553ea3f2270942959e78194b90559c8f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/553ea3f2270942959e78194b90559c8f 2024-11-11T03:40:47,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6e126d23c35b47d885b9dc8760d643c8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6e126d23c35b47d885b9dc8760d643c8 2024-11-11T03:40:47,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/f7625d34419f4670a86eaf482f99a7f9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/f7625d34419f4670a86eaf482f99a7f9 2024-11-11T03:40:47,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/a17a7de85fe64585801da8604f385064 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/a17a7de85fe64585801da8604f385064 2024-11-11T03:40:47,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8c0339df2f6b4799a1bdb36338b749c4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8c0339df2f6b4799a1bdb36338b749c4 2024-11-11T03:40:47,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cbf53d5eabc540a5bb30dcdeff546890 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cbf53d5eabc540a5bb30dcdeff546890 2024-11-11T03:40:47,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/e3d2a7b06e104577a82d997fab2392ed to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/e3d2a7b06e104577a82d997fab2392ed 2024-11-11T03:40:47,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cfcca77560b5440e8af1251214edd132 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/cfcca77560b5440e8af1251214edd132 2024-11-11T03:40:47,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/4789111e644846caad1f0fac9af84caa to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/4789111e644846caad1f0fac9af84caa 2024-11-11T03:40:47,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b7e3f71238164ada92389e8b704adec1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b7e3f71238164ada92389e8b704adec1 2024-11-11T03:40:47,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/329437a494194265869181fda9347fcc to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/329437a494194265869181fda9347fcc 2024-11-11T03:40:47,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b86b09e893de4b048d31d9811bb5b883 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/b86b09e893de4b048d31d9811bb5b883 2024-11-11T03:40:47,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/fa8b4008ac3544588983756f1d6496ce to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/fa8b4008ac3544588983756f1d6496ce 2024-11-11T03:40:47,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1abd50d4956643fcba6a72d321ff8d97 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1abd50d4956643fcba6a72d321ff8d97 2024-11-11T03:40:47,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/859e10e3816c4456843f0e74c4a07100 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/859e10e3816c4456843f0e74c4a07100 2024-11-11T03:40:47,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/0875c7740568481ab8bfaf42b6a646ee to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/0875c7740568481ab8bfaf42b6a646ee 2024-11-11T03:40:47,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/69dec9fe52dd41daa8987e8071413620 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/69dec9fe52dd41daa8987e8071413620 2024-11-11T03:40:47,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6805ba34788a443987aa1531eb140fc7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/6805ba34788a443987aa1531eb140fc7 2024-11-11T03:40:47,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/05a678a1618b41afaec2bba3edc57fd9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/05a678a1618b41afaec2bba3edc57fd9 2024-11-11T03:40:47,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/72a3a570035f4352a318597e3adf9d27 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/72a3a570035f4352a318597e3adf9d27 2024-11-11T03:40:47,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/71b8bb7311e14275bf6bc2827e64f627 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/71b8bb7311e14275bf6bc2827e64f627 2024-11-11T03:40:47,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8a350b2c34804ea1afa7c654a5134084 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/8a350b2c34804ea1afa7c654a5134084 2024-11-11T03:40:47,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/791d02d07c0341d688c71c7c495b69f2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/791d02d07c0341d688c71c7c495b69f2 2024-11-11T03:40:47,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1475e388417b4d019122e448f3b5d0d8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/1475e388417b4d019122e448f3b5d0d8 2024-11-11T03:40:47,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/76e85f0752664040b38548c563d27bb4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/76e85f0752664040b38548c563d27bb4 2024-11-11T03:40:47,776 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/recovered.edits/422.seqid, newMaxSeqId=422, maxSeqId=4 2024-11-11T03:40:47,777 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. 2024-11-11T03:40:47,777 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 887247b757faef0e5f5ee791239fb929: Waiting for close lock at 1731296445947Running coprocessor pre-close hooks at 1731296445947Disabling compacts and flushes for region at 1731296445947Disabling writes for close at 1731296445948 (+1 ms)Obtaining lock to block concurrent updates at 1731296445948Preparing flush snapshotting stores in 887247b757faef0e5f5ee791239fb929 at 1731296445948Finished memstore snapshotting TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731296445949 (+1 ms)Flushing stores of TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929. at 1731296445951 (+2 ms)Flushing 887247b757faef0e5f5ee791239fb929/A: creating writer at 1731296445951Flushing mob file 887247b757faef0e5f5ee791239fb929/A: appending metadata at 1731296445959 (+8 ms)Flushing mob file 887247b757faef0e5f5ee791239fb929/A: closing flushed file at 1731296445959Flushing 887247b757faef0e5f5ee791239fb929/A: appending metadata at 1731296446374 (+415 ms)Flushing 887247b757faef0e5f5ee791239fb929/A: closing flushed file at 1731296446374Flushing 887247b757faef0e5f5ee791239fb929/B: creating writer at 1731296446792 (+418 ms)Flushing 887247b757faef0e5f5ee791239fb929/B: appending metadata at 1731296446796 (+4 ms)Flushing 887247b757faef0e5f5ee791239fb929/B: closing flushed file at 1731296446796Flushing 887247b757faef0e5f5ee791239fb929/C: creating writer at 1731296447248 (+452 ms)Flushing 887247b757faef0e5f5ee791239fb929/C: appending metadata at 1731296447252 (+4 ms)Flushing 887247b757faef0e5f5ee791239fb929/C: closing flushed file at 1731296447252Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27d1982f: reopening flushed file at 1731296447662 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eccc0f8: reopening flushed file at 1731296447669 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@622797f6: reopening flushed file at 1731296447675 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 887247b757faef0e5f5ee791239fb929 in 1733ms, sequenceid=419, compaction requested=true at 1731296447681 (+6 ms)Writing region close event to WAL at 1731296447773 (+92 ms)Running coprocessor post-close hooks at 1731296447777 (+4 ms)Closed at 1731296447777 2024-11-11T03:40:47,779 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=887247b757faef0e5f5ee791239fb929, regionState=CLOSED 2024-11-11T03:40:47,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:40:47,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T03:40:47,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 887247b757faef0e5f5ee791239fb929, server=3ffe7f9d089e,34473,1731296384288 in 1.9890 sec 2024-11-11T03:40:47,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-11T03:40:47,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=887247b757faef0e5f5ee791239fb929, UNASSIGN in 1.9950 sec 2024-11-11T03:40:47,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T03:40:47,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9990 sec 2024-11-11T03:40:47,787 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296447787"}]},"ts":"1731296447787"} 2024-11-11T03:40:47,788 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T03:40:47,788 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T03:40:47,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0120 sec 2024-11-11T03:40:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T03:40:47,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:47,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T03:40:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:47,938 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T03:40:47,941 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:47,945 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,948 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/recovered.edits] 2024-11-11T03:40:47,952 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0fe39168105a429da6d89beafbb66c4a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/0fe39168105a429da6d89beafbb66c4a 2024-11-11T03:40:47,954 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/6ee3129e0d6e4367b9c186e6c6d6cbaf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/6ee3129e0d6e4367b9c186e6c6d6cbaf 2024-11-11T03:40:47,956 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/73d351a75f2b45a19d28f09ff02fa396 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/A/73d351a75f2b45a19d28f09ff02fa396 2024-11-11T03:40:47,960 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/20f4aacc4f464799bed13cb3ae512be4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/20f4aacc4f464799bed13cb3ae512be4 2024-11-11T03:40:47,962 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7ee6124d845d476dbe1529ff7d283a95 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/7ee6124d845d476dbe1529ff7d283a95 2024-11-11T03:40:47,964 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9ae301ba30194e8baab4c39ac3f0c2bc to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/B/9ae301ba30194e8baab4c39ac3f0c2bc 2024-11-11T03:40:47,966 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/3ef15184702c4952b1431478269cc83d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/3ef15184702c4952b1431478269cc83d 2024-11-11T03:40:47,967 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/873c912c5b5f41bc894a490f1c86ac62 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/873c912c5b5f41bc894a490f1c86ac62 2024-11-11T03:40:47,968 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/c139c1b27d094faf8a856fef597d17b1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/C/c139c1b27d094faf8a856fef597d17b1 2024-11-11T03:40:47,971 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/recovered.edits/422.seqid to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929/recovered.edits/422.seqid 2024-11-11T03:40:47,972 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,972 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T03:40:47,972 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T03:40:47,973 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T03:40:47,977 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111018825e4a824c50b94bc2b7062b187c_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111018825e4a824c50b94bc2b7062b187c_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,978 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111129a66bfbacde4e70a67e6a0435fa72b7_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111129a66bfbacde4e70a67e6a0435fa72b7_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,979 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b679fe2cae14a53bf1e57b927ef10fb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b679fe2cae14a53bf1e57b927ef10fb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,980 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113492de9592674351a82bce5c748697ab_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113492de9592674351a82bce5c748697ab_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,981 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b901c4e4b1e4874ad9d15df26d277d9_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b901c4e4b1e4874ad9d15df26d277d9_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,983 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154cb02f029cd4d7d91626e29a342d502_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111154cb02f029cd4d7d91626e29a342d502_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,984 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156b776cd2a204cdcba53d0a1dd2bf7e3_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111156b776cd2a204cdcba53d0a1dd2bf7e3_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,985 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111162db4e778ae4498286cd01abc9a68ab3_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111162db4e778ae4498286cd01abc9a68ab3_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,986 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116dadfd02d6a84b04b21858c8f479a8eb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116dadfd02d6a84b04b21858c8f479a8eb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,988 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111706d55abba3f4a3bac151111545465c5_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111706d55abba3f4a3bac151111545465c5_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,988 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ea104b17cc1476398894086b6e52488_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ea104b17cc1476398894086b6e52488_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,989 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111989dc6d81fbc41f68a0282fa732c1e7a_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111989dc6d81fbc41f68a0282fa732c1e7a_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,991 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119be260bb394d49b9adcc5ac8ac8c0590_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119be260bb394d49b9adcc5ac8ac8c0590_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,992 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119e03258cd091474bbca66f0493f4d186_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119e03258cd091474bbca66f0493f4d186_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,993 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a37e1304c9c44d948f77809213cb4977_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a37e1304c9c44d948f77809213cb4977_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,994 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111af81071734de41fd9dd99e83104857fb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111af81071734de41fd9dd99e83104857fb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,995 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c508b69604ec4dc394cbcabd42740c2a_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c508b69604ec4dc394cbcabd42740c2a_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,996 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cd493350eee44270b4878567cbbe9494_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cd493350eee44270b4878567cbbe9494_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,997 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d0e118a69bf1489490b023e1a4a33705_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d0e118a69bf1489490b023e1a4a33705_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,998 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d898a7341ece45a4a74cad117077a609_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d898a7341ece45a4a74cad117077a609_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:47,999 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dd150e2e89a2425c94c19f3208f1a4fb_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dd150e2e89a2425c94c19f3208f1a4fb_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:48,001 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e37e16363677465ab64f3b5531abd9d2_887247b757faef0e5f5ee791239fb929 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e37e16363677465ab64f3b5531abd9d2_887247b757faef0e5f5ee791239fb929 2024-11-11T03:40:48,001 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T03:40:48,003 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:48,005 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T03:40:48,007 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T03:40:48,009 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:48,009 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T03:40:48,009 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731296448009"}]},"ts":"9223372036854775807"} 2024-11-11T03:40:48,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T03:40:48,011 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 887247b757faef0e5f5ee791239fb929, NAME => 'TestAcidGuarantees,,1731296418882.887247b757faef0e5f5ee791239fb929.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T03:40:48,011 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T03:40:48,011 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731296448011"}]},"ts":"9223372036854775807"} 2024-11-11T03:40:48,013 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T03:40:48,014 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:48,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-11T03:40:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T03:40:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T03:40:48,052 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:48,063 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1022135714_22 at /127.0.0.1:39294 [Waiting for operation #1020] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383580093_22 at /127.0.0.1:60448 [Waiting for operation #1435] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383580093_22 at /127.0.0.1:58510 [Waiting for operation #1159] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1022135714_22 at /127.0.0.1:39292 [Waiting for operation #1023] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 278) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11967 (was 12088) 2024-11-11T03:40:48,075 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=11967 2024-11-11T03:40:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:40:48,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:40:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T03:40:48,079 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T03:40:48,079 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:48,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-11T03:40:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T03:40:48,080 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T03:40:48,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742076_1252 (size=1061) 2024-11-11T03:40:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T03:40:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T03:40:48,490 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:40:48,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742077_1253 (size=53) 2024-11-11T03:40:48,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:40:48,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae42fca93718bdb92e7acde763490f33, disabling compactions & flushes 2024-11-11T03:40:48,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. after waiting 0 ms 2024-11-11T03:40:48,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae42fca93718bdb92e7acde763490f33: Waiting for close lock at 1731296448499Disabling compacts and flushes for region at 1731296448499Disabling writes for close at 1731296448499Writing region close event to WAL at 1731296448499Closed at 1731296448499 2024-11-11T03:40:48,500 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T03:40:48,500 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731296448500"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731296448500"}]},"ts":"1731296448500"} 2024-11-11T03:40:48,501 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T03:40:48,502 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T03:40:48,502 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296448502"}]},"ts":"1731296448502"} 2024-11-11T03:40:48,503 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T03:40:48,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, ASSIGN}] 2024-11-11T03:40:48,504 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, ASSIGN 2024-11-11T03:40:48,505 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, ASSIGN; state=OFFLINE, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=false 2024-11-11T03:40:48,655 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=ae42fca93718bdb92e7acde763490f33, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:48,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, ASSIGN because future has completed 2024-11-11T03:40:48,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:40:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T03:40:48,816 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,816 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:40:48,817 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,817 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:40:48,817 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,817 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,821 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,823 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:48,823 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae42fca93718bdb92e7acde763490f33 columnFamilyName A 2024-11-11T03:40:48,823 DEBUG [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:48,824 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.HStore(327): Store=ae42fca93718bdb92e7acde763490f33/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:48,824 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,826 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:48,826 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae42fca93718bdb92e7acde763490f33 columnFamilyName B 2024-11-11T03:40:48,826 DEBUG [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:48,826 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.HStore(327): Store=ae42fca93718bdb92e7acde763490f33/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:48,826 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,828 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:40:48,828 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae42fca93718bdb92e7acde763490f33 columnFamilyName C 2024-11-11T03:40:48,828 DEBUG [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:40:48,829 INFO [StoreOpener-ae42fca93718bdb92e7acde763490f33-1 {}] regionserver.HStore(327): Store=ae42fca93718bdb92e7acde763490f33/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:40:48,829 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,829 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,830 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,830 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,832 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,832 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,832 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:40:48,834 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,836 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:40:48,837 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened ae42fca93718bdb92e7acde763490f33; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72425468, jitterRate=0.07922357320785522}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:40:48,837 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:48,837 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for ae42fca93718bdb92e7acde763490f33: Running coprocessor pre-open hook at 1731296448817Writing region info on filesystem at 1731296448817Initializing all the Stores at 1731296448820 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296448820Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296448820Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296448820Cleaning up temporary data from old regions at 1731296448832 (+12 ms)Running coprocessor post-open hooks at 1731296448837 (+5 ms)Region opened successfully at 1731296448837 2024-11-11T03:40:48,838 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., pid=65, masterSystemTime=1731296448809 2024-11-11T03:40:48,840 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,840 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:48,841 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=ae42fca93718bdb92e7acde763490f33, regionState=OPEN, openSeqNum=2, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:48,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:40:48,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T03:40:48,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 in 187 msec 2024-11-11T03:40:48,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-11T03:40:48,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, ASSIGN in 344 msec 2024-11-11T03:40:48,850 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T03:40:48,850 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296448850"}]},"ts":"1731296448850"} 2024-11-11T03:40:48,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T03:40:48,852 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T03:40:48,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 775 msec 2024-11-11T03:40:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T03:40:49,223 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:49,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,229 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c0421d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dede020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,234 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T03:40:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ffe7f9d089e,45295,1731296383499 2024-11-11T03:40:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@dac2b81 2024-11-11T03:40:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:40:49,239 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:40:49,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,242 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ea7c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,244 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d56d066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c067da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,248 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e808c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a290884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@316ce87e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b21ba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab85e89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,257 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b01d267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,259 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34970959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,260 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4491cf7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,261 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c940fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,263 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6b2025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,264 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa2b37a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,266 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4411834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,267 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c683bb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,269 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797bc998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,269 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595d2ee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:40:49,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:40:49,271 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:40:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:40:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:40:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fdbbd86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:40:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:40:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:40:49,272 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:40:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab7ef25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:40:49,273 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,274 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,274 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,274 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,274 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,274 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:49,275 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,275 DEBUG [Thread-1176 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,275 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,275 DEBUG [Thread-1181 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,275 DEBUG [Thread-1178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T03:40:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,276 DEBUG [Thread-1187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:40:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,277 DEBUG [Thread-1181 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,277 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:49,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T03:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,279 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:49,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:40:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:40:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:40:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:40:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:49,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T03:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:49,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/6a2824560d4440c5b0df4f868faa62a4 is 50, key is test_row_0/A:col10/1731296449287/Put/seqid=0 2024-11-11T03:40:49,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742078_1254 (size=14341) 2024-11-11T03:40:49,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296509324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296509324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296509324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296509325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296509328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T03:40:49,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T03:40:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296509442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296509442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296509442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296509442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296509442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T03:40:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T03:40:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296509652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296509652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296509652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296509652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296509653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/6a2824560d4440c5b0df4f868faa62a4 2024-11-11T03:40:49,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T03:40:49,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:49,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/4d858faf3baa4fb6a70409059ae8fa41 is 50, key is test_row_0/B:col10/1731296449287/Put/seqid=0 2024-11-11T03:40:49,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742079_1255 (size=12001) 2024-11-11T03:40:49,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/4d858faf3baa4fb6a70409059ae8fa41 2024-11-11T03:40:49,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/0da54b8209bd49e18353c945bc96888b is 50, key is test_row_0/C:col10/1731296449287/Put/seqid=0 2024-11-11T03:40:49,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742080_1256 (size=12001) 2024-11-11T03:40:49,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T03:40:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:49,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T03:40:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296509962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296509962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296509962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296509962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:49,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296509963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T03:40:50,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:50,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:50,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/0da54b8209bd49e18353c945bc96888b 2024-11-11T03:40:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/6a2824560d4440c5b0df4f868faa62a4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/6a2824560d4440c5b0df4f868faa62a4 2024-11-11T03:40:50,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/6a2824560d4440c5b0df4f868faa62a4, entries=200, sequenceid=13, filesize=14.0 K 2024-11-11T03:40:50,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/4d858faf3baa4fb6a70409059ae8fa41 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/4d858faf3baa4fb6a70409059ae8fa41 2024-11-11T03:40:50,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T03:40:50,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:50,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:50,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:50,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:50,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/4d858faf3baa4fb6a70409059ae8fa41, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T03:40:50,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/0da54b8209bd49e18353c945bc96888b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/0da54b8209bd49e18353c945bc96888b 2024-11-11T03:40:50,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/0da54b8209bd49e18353c945bc96888b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T03:40:50,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae42fca93718bdb92e7acde763490f33 in 923ms, sequenceid=13, compaction requested=false 2024-11-11T03:40:50,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:50,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T03:40:50,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:50,352 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T03:40:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:50,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0fcdbadc8f8248b589356f6b0b8d6383 is 50, key is test_row_0/A:col10/1731296449324/Put/seqid=0 2024-11-11T03:40:50,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742081_1257 (size=12001) 2024-11-11T03:40:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T03:40:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:50,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296510476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296510479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296510480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296510481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296510482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296510592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296510592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296510592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296510592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296510592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,765 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0fcdbadc8f8248b589356f6b0b8d6383 2024-11-11T03:40:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5c4134e780784816a98802368b7b62f4 is 50, key is test_row_0/B:col10/1731296449324/Put/seqid=0 2024-11-11T03:40:50,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742082_1258 (size=12001) 2024-11-11T03:40:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296510802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296510802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296510802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296510802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296510802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296511112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296511112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296511112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296511112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296511112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,176 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5c4134e780784816a98802368b7b62f4 2024-11-11T03:40:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/05aef7a8f7b24ffb98f003996e38a5e7 is 50, key is test_row_0/C:col10/1731296449324/Put/seqid=0 2024-11-11T03:40:51,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742083_1259 (size=12001) 2024-11-11T03:40:51,199 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/05aef7a8f7b24ffb98f003996e38a5e7 2024-11-11T03:40:51,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0fcdbadc8f8248b589356f6b0b8d6383 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0fcdbadc8f8248b589356f6b0b8d6383 2024-11-11T03:40:51,207 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0fcdbadc8f8248b589356f6b0b8d6383, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T03:40:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5c4134e780784816a98802368b7b62f4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c4134e780784816a98802368b7b62f4 2024-11-11T03:40:51,211 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c4134e780784816a98802368b7b62f4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T03:40:51,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/05aef7a8f7b24ffb98f003996e38a5e7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/05aef7a8f7b24ffb98f003996e38a5e7 2024-11-11T03:40:51,215 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/05aef7a8f7b24ffb98f003996e38a5e7, entries=150, sequenceid=33, filesize=11.7 K 2024-11-11T03:40:51,216 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae42fca93718bdb92e7acde763490f33 in 864ms, sequenceid=33, compaction requested=false 2024-11-11T03:40:51,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:51,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:51,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T03:40:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T03:40:51,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T03:40:51,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-11-11T03:40:51,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9460 sec 2024-11-11T03:40:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T03:40:51,432 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:51,432 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T03:40:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T03:40:51,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:51,435 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:51,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T03:40:51,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T03:40:51,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:51,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T03:40:51,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:51,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:51,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:51,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:51,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:51,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/f5a6835da2514c6c8653e85a9fa1fd69 is 50, key is test_row_0/A:col10/1731296450474/Put/seqid=0 2024-11-11T03:40:51,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742084_1260 (size=12001) 2024-11-11T03:40:51,601 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/f5a6835da2514c6c8653e85a9fa1fd69 2024-11-11T03:40:51,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/fd29aae11c0a46089016bb68b69b3dd2 is 50, key is test_row_0/B:col10/1731296450474/Put/seqid=0 2024-11-11T03:40:51,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742085_1261 (size=12001) 2024-11-11T03:40:51,616 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/fd29aae11c0a46089016bb68b69b3dd2 2024-11-11T03:40:51,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/cb349945b1e342c88676fbade439d51d is 50, key is test_row_0/C:col10/1731296450474/Put/seqid=0 2024-11-11T03:40:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:51,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742086_1262 (size=12001) 2024-11-11T03:40:51,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296511640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296511641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296511642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296511643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296511647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T03:40:51,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296511752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296511752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296511752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296511752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296511762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296511962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296511962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296511962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296511962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296511972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,029 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/cb349945b1e342c88676fbade439d51d 2024-11-11T03:40:52,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/f5a6835da2514c6c8653e85a9fa1fd69 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f5a6835da2514c6c8653e85a9fa1fd69 2024-11-11T03:40:52,036 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f5a6835da2514c6c8653e85a9fa1fd69, entries=150, sequenceid=45, filesize=11.7 K 2024-11-11T03:40:52,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/fd29aae11c0a46089016bb68b69b3dd2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/fd29aae11c0a46089016bb68b69b3dd2 2024-11-11T03:40:52,040 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/fd29aae11c0a46089016bb68b69b3dd2, entries=150, sequenceid=45, filesize=11.7 K 2024-11-11T03:40:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/cb349945b1e342c88676fbade439d51d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/cb349945b1e342c88676fbade439d51d 2024-11-11T03:40:52,045 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/cb349945b1e342c88676fbade439d51d, entries=150, sequenceid=45, filesize=11.7 K 2024-11-11T03:40:52,046 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae42fca93718bdb92e7acde763490f33 in 459ms, sequenceid=45, compaction requested=true 2024-11-11T03:40:52,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:52,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:52,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T03:40:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T03:40:52,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T03:40:52,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 613 msec 2024-11-11T03:40:52,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 619 msec 2024-11-11T03:40:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T03:40:52,062 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:52,063 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T03:40:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T03:40:52,065 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:52,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:52,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T03:40:52,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T03:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:52,220 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T03:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:52,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/8e8793156f21432288f02c037d46bb7e is 50, key is test_row_0/A:col10/1731296451639/Put/seqid=0 2024-11-11T03:40:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:52,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:52,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742087_1263 (size=12001) 2024-11-11T03:40:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296512285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296512286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296512287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296512289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296512289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T03:40:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296512402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296512402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296512402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296512402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296512402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296512612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296512612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296512612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296512612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296512612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,681 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/8e8793156f21432288f02c037d46bb7e 2024-11-11T03:40:52,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/f19662267c004c96aaf154065c218712 is 50, key is test_row_0/B:col10/1731296451639/Put/seqid=0 2024-11-11T03:40:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T03:40:52,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742088_1264 (size=12001) 2024-11-11T03:40:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296512922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296512922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296512922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296512922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296512922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,096 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/f19662267c004c96aaf154065c218712 2024-11-11T03:40:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/5f6a7227006d4e8aaa9ce72ba0d902ae is 50, key is test_row_0/C:col10/1731296451639/Put/seqid=0 2024-11-11T03:40:53,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742089_1265 (size=12001) 2024-11-11T03:40:53,116 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/5f6a7227006d4e8aaa9ce72ba0d902ae 2024-11-11T03:40:53,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/8e8793156f21432288f02c037d46bb7e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8e8793156f21432288f02c037d46bb7e 2024-11-11T03:40:53,125 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8e8793156f21432288f02c037d46bb7e, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T03:40:53,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/f19662267c004c96aaf154065c218712 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f19662267c004c96aaf154065c218712 2024-11-11T03:40:53,129 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f19662267c004c96aaf154065c218712, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T03:40:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/5f6a7227006d4e8aaa9ce72ba0d902ae as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/5f6a7227006d4e8aaa9ce72ba0d902ae 2024-11-11T03:40:53,134 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/5f6a7227006d4e8aaa9ce72ba0d902ae, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T03:40:53,135 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae42fca93718bdb92e7acde763490f33 in 916ms, sequenceid=65, compaction requested=true 2024-11-11T03:40:53,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:53,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T03:40:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T03:40:53,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T03:40:53,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0700 sec 2024-11-11T03:40:53,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0760 sec 2024-11-11T03:40:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T03:40:53,202 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:53,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T03:40:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T03:40:53,204 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:53,205 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:53,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:53,304 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:40:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T03:40:53,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T03:40:53,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,357 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T03:40:53,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:53,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:53,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:53,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:53,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:53,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/3f2ce7c72af542c19c80447d9e1825f0 is 50, key is test_row_0/A:col10/1731296452286/Put/seqid=0 2024-11-11T03:40:53,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742090_1266 (size=12001) 2024-11-11T03:40:53,368 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/3f2ce7c72af542c19c80447d9e1825f0 2024-11-11T03:40:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/8716f77849ba40ad94cd4574ec4afb8e is 50, key is test_row_0/B:col10/1731296452286/Put/seqid=0 2024-11-11T03:40:53,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742091_1267 (size=12001) 2024-11-11T03:40:53,384 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/8716f77849ba40ad94cd4574ec4afb8e 2024-11-11T03:40:53,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/09328d304d9542e1b2eb60738dc556a9 is 50, key is test_row_0/C:col10/1731296452286/Put/seqid=0 2024-11-11T03:40:53,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742092_1268 (size=12001) 2024-11-11T03:40:53,403 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/09328d304d9542e1b2eb60738dc556a9 2024-11-11T03:40:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/3f2ce7c72af542c19c80447d9e1825f0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/3f2ce7c72af542c19c80447d9e1825f0 2024-11-11T03:40:53,415 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/3f2ce7c72af542c19c80447d9e1825f0, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T03:40:53,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/8716f77849ba40ad94cd4574ec4afb8e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8716f77849ba40ad94cd4574ec4afb8e 2024-11-11T03:40:53,423 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8716f77849ba40ad94cd4574ec4afb8e, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T03:40:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/09328d304d9542e1b2eb60738dc556a9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/09328d304d9542e1b2eb60738dc556a9 2024-11-11T03:40:53,430 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/09328d304d9542e1b2eb60738dc556a9, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T03:40:53,430 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for ae42fca93718bdb92e7acde763490f33 in 73ms, sequenceid=77, compaction requested=true 2024-11-11T03:40:53,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:53,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T03:40:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T03:40:53,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T03:40:53,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 228 msec 2024-11-11T03:40:53,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 233 msec 2024-11-11T03:40:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:53,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:40:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:53,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/e66e0842732b49229c0a803a7a9ef112 is 50, key is test_row_1/A:col10/1731296453437/Put/seqid=0 2024-11-11T03:40:53,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742093_1269 (size=14337) 2024-11-11T03:40:53,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296453457, max timestamp=1731296453463Num uniques 859; 2024-11-11T03:40:53,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296453457, max timestamp=1731296453463Num uniques 859; 2024-11-11T03:40:53,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:40:53,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296453451, max timestamp=1731296453463Num uniques 859; 2024-11-11T03:40:53,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:40:53,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:40:53,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296513481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296513483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296513483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296513484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296513488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T03:40:53,522 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:53,522 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T03:40:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T03:40:53,524 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:53,525 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:53,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:53,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296513592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296513592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296513592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296513592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296513602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T03:40:53,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296513801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296513802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296513802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296513802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296513812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:53,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:53,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:53,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T03:40:53,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/e66e0842732b49229c0a803a7a9ef112 2024-11-11T03:40:53,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/1b36d0dd769e4a9d911e97fdb529d736 is 50, key is test_row_1/B:col10/1731296453437/Put/seqid=0 2024-11-11T03:40:53,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742094_1270 (size=9657) 2024-11-11T03:40:53,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:53,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296514112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296514112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296514112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296514112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296514122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:54,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:54,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T03:40:54,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/1b36d0dd769e4a9d911e97fdb529d736 2024-11-11T03:40:54,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/ff071ec8d0d645bbab9c198d0bc552eb is 50, key is test_row_1/C:col10/1731296453437/Put/seqid=0 2024-11-11T03:40:54,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742095_1271 (size=9657) 2024-11-11T03:40:54,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:54,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:54,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:54,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:54,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296514621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296514621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296514621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296514622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296514632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T03:40:54,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/ff071ec8d0d645bbab9c198d0bc552eb 2024-11-11T03:40:54,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/e66e0842732b49229c0a803a7a9ef112 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/e66e0842732b49229c0a803a7a9ef112 2024-11-11T03:40:54,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/e66e0842732b49229c0a803a7a9ef112, entries=200, sequenceid=87, filesize=14.0 K 2024-11-11T03:40:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/1b36d0dd769e4a9d911e97fdb529d736 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/1b36d0dd769e4a9d911e97fdb529d736 2024-11-11T03:40:54,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/1b36d0dd769e4a9d911e97fdb529d736, entries=100, sequenceid=87, filesize=9.4 K 2024-11-11T03:40:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/ff071ec8d0d645bbab9c198d0bc552eb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ff071ec8d0d645bbab9c198d0bc552eb 2024-11-11T03:40:54,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ff071ec8d0d645bbab9c198d0bc552eb, entries=100, sequenceid=87, filesize=9.4 K 2024-11-11T03:40:54,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for ae42fca93718bdb92e7acde763490f33 in 1250ms, sequenceid=87, compaction requested=true 2024-11-11T03:40:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:54,688 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T03:40:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:54,688 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T03:40:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:54,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:54,690 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 69662 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T03:40:54,690 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 76682 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T03:40:54,690 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:40:54,690 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:40:54,690 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,690 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,690 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/4d858faf3baa4fb6a70409059ae8fa41, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c4134e780784816a98802368b7b62f4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/fd29aae11c0a46089016bb68b69b3dd2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f19662267c004c96aaf154065c218712, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8716f77849ba40ad94cd4574ec4afb8e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/1b36d0dd769e4a9d911e97fdb529d736] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=68.0 K 2024-11-11T03:40:54,691 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/6a2824560d4440c5b0df4f868faa62a4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0fcdbadc8f8248b589356f6b0b8d6383, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f5a6835da2514c6c8653e85a9fa1fd69, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8e8793156f21432288f02c037d46bb7e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/3f2ce7c72af542c19c80447d9e1825f0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/e66e0842732b49229c0a803a7a9ef112] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=74.9 K 2024-11-11T03:40:54,691 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d858faf3baa4fb6a70409059ae8fa41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731296449287 2024-11-11T03:40:54,691 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a2824560d4440c5b0df4f868faa62a4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731296449287 2024-11-11T03:40:54,691 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fcdbadc8f8248b589356f6b0b8d6383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731296449316 2024-11-11T03:40:54,691 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c4134e780784816a98802368b7b62f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731296449316 2024-11-11T03:40:54,692 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a6835da2514c6c8653e85a9fa1fd69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731296450474 2024-11-11T03:40:54,692 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting fd29aae11c0a46089016bb68b69b3dd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731296450474 2024-11-11T03:40:54,692 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f19662267c004c96aaf154065c218712, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731296451639 2024-11-11T03:40:54,692 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e8793156f21432288f02c037d46bb7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731296451639 2024-11-11T03:40:54,692 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8716f77849ba40ad94cd4574ec4afb8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731296452286 2024-11-11T03:40:54,692 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f2ce7c72af542c19c80447d9e1825f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731296452286 2024-11-11T03:40:54,693 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b36d0dd769e4a9d911e97fdb529d736, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731296453436 2024-11-11T03:40:54,693 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66e0842732b49229c0a803a7a9ef112, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731296453434 2024-11-11T03:40:54,704 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#224 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:54,705 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/2bd3746ddf2d41638a803e69af9cae61 is 50, key is test_row_0/A:col10/1731296452286/Put/seqid=0 2024-11-11T03:40:54,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742096_1272 (size=12207) 2024-11-11T03:40:54,711 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:54,712 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/aee8d745deb64243b3d0288106f79a3c is 50, key is test_row_0/B:col10/1731296452286/Put/seqid=0 2024-11-11T03:40:54,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742097_1273 (size=12207) 2024-11-11T03:40:54,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T03:40:54,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:54,747 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-11T03:40:54,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:54,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:54,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:54,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:54,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0ba5e6f15d1845f0ba428f00dbfa1509 is 50, key is test_row_0/A:col10/1731296453486/Put/seqid=0 2024-11-11T03:40:54,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742098_1274 (size=12001) 2024-11-11T03:40:55,115 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/2bd3746ddf2d41638a803e69af9cae61 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2bd3746ddf2d41638a803e69af9cae61 2024-11-11T03:40:55,120 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/aee8d745deb64243b3d0288106f79a3c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/aee8d745deb64243b3d0288106f79a3c 2024-11-11T03:40:55,120 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into 2bd3746ddf2d41638a803e69af9cae61(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:55,120 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:55,120 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=10, startTime=1731296454688; duration=0sec 2024-11-11T03:40:55,120 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:55,121 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:40:55,121 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T03:40:55,123 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 69662 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T03:40:55,123 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:40:55,123 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:55,123 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/0da54b8209bd49e18353c945bc96888b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/05aef7a8f7b24ffb98f003996e38a5e7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/cb349945b1e342c88676fbade439d51d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/5f6a7227006d4e8aaa9ce72ba0d902ae, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/09328d304d9542e1b2eb60738dc556a9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ff071ec8d0d645bbab9c198d0bc552eb] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=68.0 K 2024-11-11T03:40:55,123 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da54b8209bd49e18353c945bc96888b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731296449287 2024-11-11T03:40:55,124 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05aef7a8f7b24ffb98f003996e38a5e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731296449316 2024-11-11T03:40:55,125 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb349945b1e342c88676fbade439d51d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731296450474 2024-11-11T03:40:55,125 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into aee8d745deb64243b3d0288106f79a3c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:55,125 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:55,125 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=10, startTime=1731296454688; duration=0sec 2024-11-11T03:40:55,125 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:55,125 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:40:55,125 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6a7227006d4e8aaa9ce72ba0d902ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731296451639 2024-11-11T03:40:55,126 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09328d304d9542e1b2eb60738dc556a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731296452286 2024-11-11T03:40:55,126 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff071ec8d0d645bbab9c198d0bc552eb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731296453436 2024-11-11T03:40:55,136 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#227 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:55,136 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/477272de5ad74b589c9e47da5405d843 is 50, key is test_row_0/C:col10/1731296452286/Put/seqid=0 2024-11-11T03:40:55,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742099_1275 (size=12207) 2024-11-11T03:40:55,156 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0ba5e6f15d1845f0ba428f00dbfa1509 2024-11-11T03:40:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/e304293e20d74aa597e4af1702ab084a is 50, key is test_row_0/B:col10/1731296453486/Put/seqid=0 2024-11-11T03:40:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742100_1276 (size=12001) 2024-11-11T03:40:55,544 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/477272de5ad74b589c9e47da5405d843 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/477272de5ad74b589c9e47da5405d843 2024-11-11T03:40:55,548 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into 477272de5ad74b589c9e47da5405d843(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:55,548 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:55,549 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=10, startTime=1731296454688; duration=0sec 2024-11-11T03:40:55,549 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:55,549 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:40:55,567 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/e304293e20d74aa597e4af1702ab084a 2024-11-11T03:40:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/2adb5050e7244c79aa968ea4bf47761b is 50, key is test_row_0/C:col10/1731296453486/Put/seqid=0 2024-11-11T03:40:55,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742101_1277 (size=12001) 2024-11-11T03:40:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:55,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:55,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296515643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296515645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296515646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296515646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296515652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,656 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T03:40:55,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296515752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296515762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296515762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296515762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296515962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296515972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296515972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296515972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:55,979 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/2adb5050e7244c79aa968ea4bf47761b 2024-11-11T03:40:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0ba5e6f15d1845f0ba428f00dbfa1509 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0ba5e6f15d1845f0ba428f00dbfa1509 2024-11-11T03:40:55,985 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0ba5e6f15d1845f0ba428f00dbfa1509, entries=150, sequenceid=123, filesize=11.7 K 2024-11-11T03:40:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/e304293e20d74aa597e4af1702ab084a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/e304293e20d74aa597e4af1702ab084a 2024-11-11T03:40:55,989 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/e304293e20d74aa597e4af1702ab084a, entries=150, sequenceid=123, filesize=11.7 K 2024-11-11T03:40:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/2adb5050e7244c79aa968ea4bf47761b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/2adb5050e7244c79aa968ea4bf47761b 2024-11-11T03:40:55,993 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/2adb5050e7244c79aa968ea4bf47761b, entries=150, sequenceid=123, filesize=11.7 K 2024-11-11T03:40:55,994 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=53.67 KB/54960 for ae42fca93718bdb92e7acde763490f33 in 1247ms, sequenceid=123, compaction requested=false 2024-11-11T03:40:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T03:40:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T03:40:55,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T03:40:55,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4710 sec 2024-11-11T03:40:55,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.4750 sec 2024-11-11T03:40:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:56,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T03:40:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:56,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/2b45282ef34343b4963f1bdc82137c1d is 50, key is test_row_0/A:col10/1731296455642/Put/seqid=0 2024-11-11T03:40:56,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742102_1278 (size=14541) 2024-11-11T03:40:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296516297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296516298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296516299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296516302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296516412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296516412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296516412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296516412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296516622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296516622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296516622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296516622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:56,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/2b45282ef34343b4963f1bdc82137c1d 2024-11-11T03:40:56,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5c5dad7212f9429f9f2de8d8335da86b is 50, key is test_row_0/B:col10/1731296455642/Put/seqid=0 2024-11-11T03:40:56,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742103_1279 (size=12151) 2024-11-11T03:40:56,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5c5dad7212f9429f9f2de8d8335da86b 2024-11-11T03:40:56,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/6240fd2d2ce840758955ffa7672375a0 is 50, key is test_row_0/C:col10/1731296455642/Put/seqid=0 2024-11-11T03:40:56,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742104_1280 (size=12151) 2024-11-11T03:40:56,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/6240fd2d2ce840758955ffa7672375a0 2024-11-11T03:40:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/2b45282ef34343b4963f1bdc82137c1d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2b45282ef34343b4963f1bdc82137c1d 2024-11-11T03:40:56,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2b45282ef34343b4963f1bdc82137c1d, entries=200, sequenceid=138, filesize=14.2 K 2024-11-11T03:40:56,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5c5dad7212f9429f9f2de8d8335da86b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c5dad7212f9429f9f2de8d8335da86b 2024-11-11T03:40:56,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c5dad7212f9429f9f2de8d8335da86b, entries=150, sequenceid=138, filesize=11.9 K 2024-11-11T03:40:56,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/6240fd2d2ce840758955ffa7672375a0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/6240fd2d2ce840758955ffa7672375a0 2024-11-11T03:40:56,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/6240fd2d2ce840758955ffa7672375a0, entries=150, sequenceid=138, filesize=11.9 K 2024-11-11T03:40:56,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae42fca93718bdb92e7acde763490f33 in 446ms, sequenceid=138, compaction requested=true 2024-11-11T03:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:56,720 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:56,720 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:56,721 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:56,721 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:56,721 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:40:56,721 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:40:56,721 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:56,721 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:56,721 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2bd3746ddf2d41638a803e69af9cae61, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0ba5e6f15d1845f0ba428f00dbfa1509, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2b45282ef34343b4963f1bdc82137c1d] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=37.8 K 2024-11-11T03:40:56,721 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/aee8d745deb64243b3d0288106f79a3c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/e304293e20d74aa597e4af1702ab084a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c5dad7212f9429f9f2de8d8335da86b] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=35.5 K 2024-11-11T03:40:56,721 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd3746ddf2d41638a803e69af9cae61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731296452286 2024-11-11T03:40:56,721 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting aee8d745deb64243b3d0288106f79a3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731296452286 2024-11-11T03:40:56,722 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba5e6f15d1845f0ba428f00dbfa1509, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731296453482 2024-11-11T03:40:56,722 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e304293e20d74aa597e4af1702ab084a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731296453482 2024-11-11T03:40:56,722 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c5dad7212f9429f9f2de8d8335da86b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731296455642 2024-11-11T03:40:56,722 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b45282ef34343b4963f1bdc82137c1d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731296455642 2024-11-11T03:40:56,731 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:56,732 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/b84e2f60b0174e6594a7b698c7e25943 is 50, key is test_row_0/B:col10/1731296455642/Put/seqid=0 2024-11-11T03:40:56,736 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:56,736 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/c93a6c6b9a194336beb7aee9e3d507b0 is 50, key is test_row_0/A:col10/1731296455642/Put/seqid=0 2024-11-11T03:40:56,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742106_1282 (size=12459) 2024-11-11T03:40:56,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742105_1281 (size=12459) 2024-11-11T03:40:56,761 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/b84e2f60b0174e6594a7b698c7e25943 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/b84e2f60b0174e6594a7b698c7e25943 2024-11-11T03:40:56,767 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into b84e2f60b0174e6594a7b698c7e25943(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:56,767 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:56,767 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=13, startTime=1731296456720; duration=0sec 2024-11-11T03:40:56,768 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:56,768 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:40:56,768 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:56,769 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:56,769 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:40:56,769 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:56,769 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/477272de5ad74b589c9e47da5405d843, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/2adb5050e7244c79aa968ea4bf47761b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/6240fd2d2ce840758955ffa7672375a0] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=35.5 K 2024-11-11T03:40:56,770 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 477272de5ad74b589c9e47da5405d843, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731296452286 2024-11-11T03:40:56,770 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2adb5050e7244c79aa968ea4bf47761b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731296453482 2024-11-11T03:40:56,770 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6240fd2d2ce840758955ffa7672375a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731296455642 2024-11-11T03:40:56,781 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:56,781 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/9e1e2e0cdb1c41b68f0993b53b67349f is 50, key is test_row_0/C:col10/1731296455642/Put/seqid=0 2024-11-11T03:40:56,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742107_1283 (size=12459) 2024-11-11T03:40:56,828 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/9e1e2e0cdb1c41b68f0993b53b67349f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/9e1e2e0cdb1c41b68f0993b53b67349f 2024-11-11T03:40:56,842 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into 9e1e2e0cdb1c41b68f0993b53b67349f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:56,842 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:56,842 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=13, startTime=1731296456720; duration=0sec 2024-11-11T03:40:56,842 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:56,842 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:40:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:56,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296456296, max timestamp=1731296456934Num uniques 859; 2024-11-11T03:40:56,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:40:56,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296456296, max timestamp=1731296456934Num uniques 859; 2024-11-11T03:40:56,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:40:56,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296456296, max timestamp=1731296456934Num uniques 859; 2024-11-11T03:40:56,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-11T03:40:56,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:40:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:40:56,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/1eac1e4c7f5d47759d2095a843422251 is 50, key is test_row_0/A:col10/1731296456296/Put/seqid=0 2024-11-11T03:40:56,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742108_1284 (size=12151) 2024-11-11T03:40:56,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/1eac1e4c7f5d47759d2095a843422251 2024-11-11T03:40:56,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/efd5b46d822c45f9b08621ba7e63003f is 50, key is test_row_0/B:col10/1731296456296/Put/seqid=0 2024-11-11T03:40:56,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296456963, max timestamp=1731296456979Num uniques 859; 2024-11-11T03:40:56,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296456963, max timestamp=1731296456979Num uniques 859; 2024-11-11T03:40:56,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:56,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:56,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296456963, max timestamp=1731296456979Num uniques 859; 2024-11-11T03:40:56,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:40:56,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742109_1285 (size=12151) 2024-11-11T03:40:56,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/efd5b46d822c45f9b08621ba7e63003f 2024-11-11T03:40:57,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/bdb3df1fe66f46058412be97d14df134 is 50, key is test_row_0/C:col10/1731296456296/Put/seqid=0 2024-11-11T03:40:57,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296517014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296517016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742110_1286 (size=12151) 2024-11-11T03:40:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296517020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296517022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296517132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296517132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296517132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296517132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,158 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/c93a6c6b9a194336beb7aee9e3d507b0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c93a6c6b9a194336beb7aee9e3d507b0 2024-11-11T03:40:57,162 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into c93a6c6b9a194336beb7aee9e3d507b0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:57,162 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:57,162 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=13, startTime=1731296456720; duration=0sec 2024-11-11T03:40:57,162 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:57,162 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:40:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296517342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296517342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296517342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296517342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/bdb3df1fe66f46058412be97d14df134 2024-11-11T03:40:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/1eac1e4c7f5d47759d2095a843422251 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1eac1e4c7f5d47759d2095a843422251 2024-11-11T03:40:57,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1eac1e4c7f5d47759d2095a843422251, entries=150, sequenceid=164, filesize=11.9 K 2024-11-11T03:40:57,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/efd5b46d822c45f9b08621ba7e63003f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/efd5b46d822c45f9b08621ba7e63003f 2024-11-11T03:40:57,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/efd5b46d822c45f9b08621ba7e63003f, entries=150, sequenceid=164, filesize=11.9 K 2024-11-11T03:40:57,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/bdb3df1fe66f46058412be97d14df134 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bdb3df1fe66f46058412be97d14df134 2024-11-11T03:40:57,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bdb3df1fe66f46058412be97d14df134, entries=150, sequenceid=164, filesize=11.9 K 2024-11-11T03:40:57,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.95 KB/128976, currentSize=140.89 KB/144270 for ae42fca93718bdb92e7acde763490f33 in 504ms, sequenceid=164, compaction requested=false 2024-11-11T03:40:57,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:57,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T03:40:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:40:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:40:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:40:57,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/776021643eda45cfa56f0aa3abf023c6 is 50, key is test_row_0/A:col10/1731296457019/Put/seqid=0 2024-11-11T03:40:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296517662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296517662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296517663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296517664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742111_1287 (size=14541) 2024-11-11T03:40:57,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296517671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,675 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T03:40:57,692 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:57,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T03:40:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T03:40:57,694 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:57,694 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:57,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:57,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296517771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296517772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296517772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296517772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T03:40:57,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:57,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:57,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:57,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:57,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296517981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296517981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296517982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296517982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:57,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:57,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:57,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:57,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:57,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:57,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T03:40:58,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/776021643eda45cfa56f0aa3abf023c6 2024-11-11T03:40:58,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/908af378c40148df973b8ebb00cd6485 is 50, key is test_row_0/B:col10/1731296457019/Put/seqid=0 2024-11-11T03:40:58,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742112_1288 (size=12151) 2024-11-11T03:40:58,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296518292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296518292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296518292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296518292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:58,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:58,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T03:40:58,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/908af378c40148df973b8ebb00cd6485 2024-11-11T03:40:58,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/890ea41fb04044a288829166f7f9a578 is 50, key is test_row_0/C:col10/1731296457019/Put/seqid=0 2024-11-11T03:40:58,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742113_1289 (size=12151) 2024-11-11T03:40:58,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296518801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296518801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296518801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296518812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T03:40:58,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/890ea41fb04044a288829166f7f9a578 2024-11-11T03:40:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/776021643eda45cfa56f0aa3abf023c6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/776021643eda45cfa56f0aa3abf023c6 2024-11-11T03:40:58,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/776021643eda45cfa56f0aa3abf023c6, entries=200, sequenceid=203, filesize=14.2 K 2024-11-11T03:40:58,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/908af378c40148df973b8ebb00cd6485 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/908af378c40148df973b8ebb00cd6485 2024-11-11T03:40:58,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/908af378c40148df973b8ebb00cd6485, entries=150, sequenceid=203, filesize=11.9 K 2024-11-11T03:40:58,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/890ea41fb04044a288829166f7f9a578 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/890ea41fb04044a288829166f7f9a578 2024-11-11T03:40:58,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/890ea41fb04044a288829166f7f9a578, entries=150, sequenceid=203, filesize=11.9 K 2024-11-11T03:40:58,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for ae42fca93718bdb92e7acde763490f33 in 1253ms, sequenceid=203, compaction requested=true 2024-11-11T03:40:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:40:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:40:58,908 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:40:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:58,908 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:40:58,909 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,909 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,909 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c93a6c6b9a194336beb7aee9e3d507b0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1eac1e4c7f5d47759d2095a843422251, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/776021643eda45cfa56f0aa3abf023c6] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=38.2 K 2024-11-11T03:40:58,909 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/b84e2f60b0174e6594a7b698c7e25943, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/efd5b46d822c45f9b08621ba7e63003f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/908af378c40148df973b8ebb00cd6485] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=35.9 K 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting c93a6c6b9a194336beb7aee9e3d507b0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731296455642 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b84e2f60b0174e6594a7b698c7e25943, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731296455642 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eac1e4c7f5d47759d2095a843422251, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731296456296 2024-11-11T03:40:58,909 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting efd5b46d822c45f9b08621ba7e63003f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731296456296 2024-11-11T03:40:58,910 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776021643eda45cfa56f0aa3abf023c6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731296457008 2024-11-11T03:40:58,910 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 908af378c40148df973b8ebb00cd6485, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731296457018 2024-11-11T03:40:58,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T03:40:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,915 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T03:40:58,915 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:58,916 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:58,916 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/8047662ae57840da97d39bf37f0f2c9f is 50, key is test_row_0/B:col10/1731296457019/Put/seqid=0 2024-11-11T03:40:58,916 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/8148c6d9a00b4af089b2dbc1d340a8f2 is 50, key is test_row_0/A:col10/1731296457019/Put/seqid=0 2024-11-11T03:40:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/ac52be863b694de6849417a1f2bc269b is 50, key is test_row_0/A:col10/1731296457656/Put/seqid=0 2024-11-11T03:40:58,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742114_1290 (size=12561) 2024-11-11T03:40:58,926 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/8148c6d9a00b4af089b2dbc1d340a8f2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8148c6d9a00b4af089b2dbc1d340a8f2 2024-11-11T03:40:58,931 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into 8148c6d9a00b4af089b2dbc1d340a8f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:58,931 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:58,931 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=13, startTime=1731296458908; duration=0sec 2024-11-11T03:40:58,931 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:40:58,931 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:40:58,931 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:40:58,932 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:40:58,932 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:40:58,932 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:58,933 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/9e1e2e0cdb1c41b68f0993b53b67349f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bdb3df1fe66f46058412be97d14df134, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/890ea41fb04044a288829166f7f9a578] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=35.9 K 2024-11-11T03:40:58,933 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e1e2e0cdb1c41b68f0993b53b67349f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731296455642 2024-11-11T03:40:58,933 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb3df1fe66f46058412be97d14df134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731296456296 2024-11-11T03:40:58,933 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890ea41fb04044a288829166f7f9a578, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731296457018 2024-11-11T03:40:58,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742115_1291 (size=12561) 2024-11-11T03:40:58,948 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:40:58,949 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/c5e5c14288234bab85f8bc10fc22e130 is 50, key is test_row_0/C:col10/1731296457019/Put/seqid=0 2024-11-11T03:40:58,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742116_1292 (size=12151) 2024-11-11T03:40:58,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742117_1293 (size=12561) 2024-11-11T03:40:59,348 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/8047662ae57840da97d39bf37f0f2c9f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8047662ae57840da97d39bf37f0f2c9f 2024-11-11T03:40:59,353 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into 8047662ae57840da97d39bf37f0f2c9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:59,354 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:59,354 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=13, startTime=1731296458908; duration=0sec 2024-11-11T03:40:59,354 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:59,354 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:40:59,356 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/ac52be863b694de6849417a1f2bc269b 2024-11-11T03:40:59,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/c30c10bf51ab45bb8e1fd29bb26bbc02 is 50, key is test_row_0/B:col10/1731296457656/Put/seqid=0 2024-11-11T03:40:59,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742118_1294 (size=12151) 2024-11-11T03:40:59,382 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/c5e5c14288234bab85f8bc10fc22e130 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c5e5c14288234bab85f8bc10fc22e130 2024-11-11T03:40:59,387 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into c5e5c14288234bab85f8bc10fc22e130(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:40:59,387 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:59,387 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=13, startTime=1731296458908; duration=0sec 2024-11-11T03:40:59,387 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:40:59,387 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:40:59,767 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/c30c10bf51ab45bb8e1fd29bb26bbc02 2024-11-11T03:40:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/ffe983282aed41a982b2eaa2cb340a6b is 50, key is test_row_0/C:col10/1731296457656/Put/seqid=0 2024-11-11T03:40:59,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742119_1295 (size=12151) 2024-11-11T03:40:59,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/ffe983282aed41a982b2eaa2cb340a6b 2024-11-11T03:40:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/ac52be863b694de6849417a1f2bc269b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/ac52be863b694de6849417a1f2bc269b 2024-11-11T03:40:59,786 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/ac52be863b694de6849417a1f2bc269b, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T03:40:59,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/c30c10bf51ab45bb8e1fd29bb26bbc02 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/c30c10bf51ab45bb8e1fd29bb26bbc02 2024-11-11T03:40:59,790 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/c30c10bf51ab45bb8e1fd29bb26bbc02, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T03:40:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/ffe983282aed41a982b2eaa2cb340a6b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ffe983282aed41a982b2eaa2cb340a6b 2024-11-11T03:40:59,796 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ffe983282aed41a982b2eaa2cb340a6b, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T03:40:59,797 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ae42fca93718bdb92e7acde763490f33 in 882ms, sequenceid=211, compaction requested=false 2024-11-11T03:40:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:40:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:59,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T03:40:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T03:40:59,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T03:40:59,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1040 sec 2024-11-11T03:40:59,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1090 sec 2024-11-11T03:40:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:40:59,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T03:40:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:40:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:40:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:40:59,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:40:59,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/73e749016e134debb6b1621dcf08643d is 50, key is test_row_0/A:col10/1731296459820/Put/seqid=0 2024-11-11T03:40:59,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742120_1296 (size=14541) 2024-11-11T03:40:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T03:40:59,842 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:40:59,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:40:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T03:40:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T03:40:59,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:40:59,846 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:40:59,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:40:59,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296519851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296519855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296519857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296519857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T03:40:59,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296519962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296519972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296519972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:40:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296519972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:40:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:40:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:40:59,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:40:59,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:59,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:40:59,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:40:59,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:40:59,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T03:41:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296520172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296520182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296520182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296520182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/73e749016e134debb6b1621dcf08643d 2024-11-11T03:41:00,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/d017b78040da429dbd8bbfa6b14ad119 is 50, key is test_row_0/B:col10/1731296459820/Put/seqid=0 2024-11-11T03:41:00,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742121_1297 (size=12151) 2024-11-11T03:41:00,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:00,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:00,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:00,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:00,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T03:41:00,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296520481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296520492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296520492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296520492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:00,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/d017b78040da429dbd8bbfa6b14ad119 2024-11-11T03:41:00,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/d1b7d84d81614f61a1aace5ae7d7c5b7 is 50, key is test_row_0/C:col10/1731296459820/Put/seqid=0 2024-11-11T03:41:00,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742122_1298 (size=12151) 2024-11-11T03:41:00,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:00,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:00,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:00,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T03:41:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296520991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296521002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:01,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296521002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:01,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296521002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:01,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/d1b7d84d81614f61a1aace5ae7d7c5b7 2024-11-11T03:41:01,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/73e749016e134debb6b1621dcf08643d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/73e749016e134debb6b1621dcf08643d 2024-11-11T03:41:01,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:01,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:01,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:01,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:01,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:01,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:01,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/73e749016e134debb6b1621dcf08643d, entries=200, sequenceid=226, filesize=14.2 K 2024-11-11T03:41:01,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/d017b78040da429dbd8bbfa6b14ad119 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d017b78040da429dbd8bbfa6b14ad119 2024-11-11T03:41:01,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d017b78040da429dbd8bbfa6b14ad119, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T03:41:01,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/d1b7d84d81614f61a1aace5ae7d7c5b7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/d1b7d84d81614f61a1aace5ae7d7c5b7 2024-11-11T03:41:01,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/d1b7d84d81614f61a1aace5ae7d7c5b7, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T03:41:01,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae42fca93718bdb92e7acde763490f33 in 1254ms, sequenceid=226, compaction requested=true 2024-11-11T03:41:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:01,079 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:01,079 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:01,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:01,080 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:01,080 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:01,080 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:41:01,080 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:41:01,080 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:01,080 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:01,080 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8047662ae57840da97d39bf37f0f2c9f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/c30c10bf51ab45bb8e1fd29bb26bbc02, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d017b78040da429dbd8bbfa6b14ad119] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.0 K 2024-11-11T03:41:01,080 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8148c6d9a00b4af089b2dbc1d340a8f2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/ac52be863b694de6849417a1f2bc269b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/73e749016e134debb6b1621dcf08643d] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=38.3 K 2024-11-11T03:41:01,081 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8047662ae57840da97d39bf37f0f2c9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731296457018 2024-11-11T03:41:01,081 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8148c6d9a00b4af089b2dbc1d340a8f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731296457018 2024-11-11T03:41:01,081 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting c30c10bf51ab45bb8e1fd29bb26bbc02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731296457656 2024-11-11T03:41:01,081 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac52be863b694de6849417a1f2bc269b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731296457656 2024-11-11T03:41:01,081 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d017b78040da429dbd8bbfa6b14ad119, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296459820 2024-11-11T03:41:01,081 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e749016e134debb6b1621dcf08643d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296459820 2024-11-11T03:41:01,088 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:01,089 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/d0acce98578f4163ae177a4186dd656f is 50, key is test_row_0/B:col10/1731296459820/Put/seqid=0 2024-11-11T03:41:01,091 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:01,092 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/33046a34f51d47df89d498b6bcfb5a4e is 50, key is test_row_0/A:col10/1731296459820/Put/seqid=0 2024-11-11T03:41:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742123_1299 (size=12663) 2024-11-11T03:41:01,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742124_1300 (size=12663) 2024-11-11T03:41:01,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T03:41:01,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:01,220 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T03:41:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:01,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/de99ee13896a435f83a33971ea81e49e is 50, key is test_row_0/A:col10/1731296459855/Put/seqid=0 2024-11-11T03:41:01,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742125_1301 (size=12151) 2024-11-11T03:41:01,498 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/d0acce98578f4163ae177a4186dd656f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d0acce98578f4163ae177a4186dd656f 2024-11-11T03:41:01,502 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into d0acce98578f4163ae177a4186dd656f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:01,502 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:01,502 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=13, startTime=1731296461079; duration=0sec 2024-11-11T03:41:01,503 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:01,503 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:41:01,503 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:01,503 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:01,503 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:41:01,504 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:01,504 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c5e5c14288234bab85f8bc10fc22e130, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ffe983282aed41a982b2eaa2cb340a6b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/d1b7d84d81614f61a1aace5ae7d7c5b7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.0 K 2024-11-11T03:41:01,504 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting c5e5c14288234bab85f8bc10fc22e130, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731296457018 2024-11-11T03:41:01,504 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe983282aed41a982b2eaa2cb340a6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731296457656 2024-11-11T03:41:01,505 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b7d84d81614f61a1aace5ae7d7c5b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296459820 2024-11-11T03:41:01,510 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/33046a34f51d47df89d498b6bcfb5a4e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/33046a34f51d47df89d498b6bcfb5a4e 2024-11-11T03:41:01,512 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:01,513 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/030358e0392c4f5a97e35df1dd04745f is 50, key is test_row_0/C:col10/1731296459820/Put/seqid=0 2024-11-11T03:41:01,516 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into 33046a34f51d47df89d498b6bcfb5a4e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:01,516 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:01,516 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=13, startTime=1731296461079; duration=0sec 2024-11-11T03:41:01,516 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:01,516 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:41:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742126_1302 (size=12663) 2024-11-11T03:41:01,522 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/030358e0392c4f5a97e35df1dd04745f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/030358e0392c4f5a97e35df1dd04745f 2024-11-11T03:41:01,526 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into 030358e0392c4f5a97e35df1dd04745f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:01,526 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:01,527 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=13, startTime=1731296461079; duration=0sec 2024-11-11T03:41:01,527 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:01,527 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:41:01,629 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/de99ee13896a435f83a33971ea81e49e 2024-11-11T03:41:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/95ee4d8560214e269f7a34201e5efa7b is 50, key is test_row_0/B:col10/1731296459855/Put/seqid=0 2024-11-11T03:41:01,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742127_1303 (size=12151) 2024-11-11T03:41:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:01,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:01,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296521745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296521852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T03:41:02,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296522001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:02,003 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:02,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:02,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296522012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:02,013 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:02,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296522022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:02,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296522022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:02,024 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,024 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:02,045 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/95ee4d8560214e269f7a34201e5efa7b 2024-11-11T03:41:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/c42a7ce1a4ec4328936ca7a6cf637cbe is 50, key is test_row_0/C:col10/1731296459855/Put/seqid=0 2024-11-11T03:41:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296522061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:02,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742128_1304 (size=12151) 2024-11-11T03:41:02,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296522372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:02,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/c42a7ce1a4ec4328936ca7a6cf637cbe 2024-11-11T03:41:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/de99ee13896a435f83a33971ea81e49e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/de99ee13896a435f83a33971ea81e49e 2024-11-11T03:41:02,541 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/de99ee13896a435f83a33971ea81e49e, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T03:41:02,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/95ee4d8560214e269f7a34201e5efa7b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/95ee4d8560214e269f7a34201e5efa7b 2024-11-11T03:41:02,545 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/95ee4d8560214e269f7a34201e5efa7b, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T03:41:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/c42a7ce1a4ec4328936ca7a6cf637cbe as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c42a7ce1a4ec4328936ca7a6cf637cbe 2024-11-11T03:41:02,550 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c42a7ce1a4ec4328936ca7a6cf637cbe, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T03:41:02,551 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae42fca93718bdb92e7acde763490f33 in 1331ms, sequenceid=246, compaction requested=false 2024-11-11T03:41:02,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:02,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:02,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T03:41:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T03:41:02,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T03:41:02,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7060 sec 2024-11-11T03:41:02,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.7110 sec 2024-11-11T03:41:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:02,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T03:41:02,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:02,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/070d9da847f04dcaa257523d1346699d is 50, key is test_row_0/A:col10/1731296461736/Put/seqid=0 2024-11-11T03:41:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742129_1305 (size=14691) 2024-11-11T03:41:02,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296522977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:03,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296523091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:03,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/070d9da847f04dcaa257523d1346699d 2024-11-11T03:41:03,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/0c8a2ec0179747baa48b0993d4947636 is 50, key is test_row_0/B:col10/1731296461736/Put/seqid=0 2024-11-11T03:41:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296523302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:03,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742130_1306 (size=12251) 2024-11-11T03:41:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296523612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:03,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/0c8a2ec0179747baa48b0993d4947636 2024-11-11T03:41:03,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/967a228140f745dbb37dea52b57442f0 is 50, key is test_row_0/C:col10/1731296461736/Put/seqid=0 2024-11-11T03:41:03,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742131_1307 (size=12251) 2024-11-11T03:41:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T03:41:04,012 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:04,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T03:41:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T03:41:04,014 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:04,015 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:04,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:04,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296524012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:04,016 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296524021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:04,024 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:04,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:04,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296524041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:04,043 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296524042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:04,045 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:04,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/967a228140f745dbb37dea52b57442f0 2024-11-11T03:41:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T03:41:04,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/070d9da847f04dcaa257523d1346699d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/070d9da847f04dcaa257523d1346699d 2024-11-11T03:41:04,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296524121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:04,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/070d9da847f04dcaa257523d1346699d, entries=200, sequenceid=262, filesize=14.3 K 2024-11-11T03:41:04,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/0c8a2ec0179747baa48b0993d4947636 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/0c8a2ec0179747baa48b0993d4947636 2024-11-11T03:41:04,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/0c8a2ec0179747baa48b0993d4947636, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T03:41:04,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/967a228140f745dbb37dea52b57442f0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/967a228140f745dbb37dea52b57442f0 2024-11-11T03:41:04,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/967a228140f745dbb37dea52b57442f0, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T03:41:04,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ae42fca93718bdb92e7acde763490f33 in 1257ms, sequenceid=262, compaction requested=true 2024-11-11T03:41:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:04,141 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:04,141 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:04,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:04,142 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:04,142 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:41:04,142 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:04,142 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d0acce98578f4163ae177a4186dd656f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/95ee4d8560214e269f7a34201e5efa7b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/0c8a2ec0179747baa48b0993d4947636] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.2 K 2024-11-11T03:41:04,143 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:04,143 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:41:04,143 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:04,143 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/33046a34f51d47df89d498b6bcfb5a4e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/de99ee13896a435f83a33971ea81e49e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/070d9da847f04dcaa257523d1346699d] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=38.6 K 2024-11-11T03:41:04,143 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d0acce98578f4163ae177a4186dd656f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296459820 2024-11-11T03:41:04,143 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33046a34f51d47df89d498b6bcfb5a4e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296459820 2024-11-11T03:41:04,143 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ee4d8560214e269f7a34201e5efa7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731296459826 2024-11-11T03:41:04,143 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting de99ee13896a435f83a33971ea81e49e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731296459826 2024-11-11T03:41:04,144 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8a2ec0179747baa48b0993d4947636, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296461717 2024-11-11T03:41:04,144 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 070d9da847f04dcaa257523d1346699d, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296461717 2024-11-11T03:41:04,150 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:04,150 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/b545b07266bf4fe4b799bd2e39cc3c95 is 50, key is test_row_0/A:col10/1731296461736/Put/seqid=0 2024-11-11T03:41:04,151 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:04,151 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/ae09bdc97c6043c7a8ff2fdbb1fe58de is 50, key is test_row_0/B:col10/1731296461736/Put/seqid=0 2024-11-11T03:41:04,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742132_1308 (size=12865) 2024-11-11T03:41:04,160 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/b545b07266bf4fe4b799bd2e39cc3c95 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/b545b07266bf4fe4b799bd2e39cc3c95 2024-11-11T03:41:04,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742133_1309 (size=12865) 2024-11-11T03:41:04,165 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into b545b07266bf4fe4b799bd2e39cc3c95(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:04,165 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:04,166 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=13, startTime=1731296464141; duration=0sec 2024-11-11T03:41:04,166 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:04,166 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:41:04,166 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:04,167 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:04,167 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:41:04,167 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:04,167 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/030358e0392c4f5a97e35df1dd04745f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c42a7ce1a4ec4328936ca7a6cf637cbe, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/967a228140f745dbb37dea52b57442f0] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.2 K 2024-11-11T03:41:04,167 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 030358e0392c4f5a97e35df1dd04745f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296459820 2024-11-11T03:41:04,168 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42a7ce1a4ec4328936ca7a6cf637cbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731296459826 2024-11-11T03:41:04,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T03:41:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:04,168 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 967a228140f745dbb37dea52b57442f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296461717 2024-11-11T03:41:04,168 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T03:41:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:04,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:04,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:04,171 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/ae09bdc97c6043c7a8ff2fdbb1fe58de as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/ae09bdc97c6043c7a8ff2fdbb1fe58de 2024-11-11T03:41:04,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0b3e6ed4cc6043dab9721e05d2d9379e is 50, key is test_row_0/A:col10/1731296462976/Put/seqid=0 2024-11-11T03:41:04,177 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into ae09bdc97c6043c7a8ff2fdbb1fe58de(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:04,177 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:04,177 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=13, startTime=1731296464141; duration=0sec 2024-11-11T03:41:04,177 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:04,177 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:04,177 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:41:04,178 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/29ac36434e8a46519595257bd6eb4ea9 is 50, key is test_row_0/C:col10/1731296461736/Put/seqid=0 2024-11-11T03:41:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742134_1310 (size=12301) 2024-11-11T03:41:04,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742135_1311 (size=12865) 2024-11-11T03:41:04,185 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/29ac36434e8a46519595257bd6eb4ea9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/29ac36434e8a46519595257bd6eb4ea9 2024-11-11T03:41:04,189 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into 29ac36434e8a46519595257bd6eb4ea9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:04,189 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:04,189 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=13, startTime=1731296464141; duration=0sec 2024-11-11T03:41:04,189 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:04,189 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:41:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T03:41:04,581 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0b3e6ed4cc6043dab9721e05d2d9379e 2024-11-11T03:41:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/9108d4a0696147ac9ecc01dd8a928440 is 50, key is test_row_0/B:col10/1731296462976/Put/seqid=0 2024-11-11T03:41:04,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742136_1312 (size=12301) 2024-11-11T03:41:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T03:41:04,991 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/9108d4a0696147ac9ecc01dd8a928440 2024-11-11T03:41:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/8483a66804d14986899d804a64ea5e08 is 50, key is test_row_0/C:col10/1731296462976/Put/seqid=0 2024-11-11T03:41:05,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742137_1313 (size=12301) 2024-11-11T03:41:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T03:41:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:05,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:05,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296525198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:05,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296525311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:05,401 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/8483a66804d14986899d804a64ea5e08 2024-11-11T03:41:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0b3e6ed4cc6043dab9721e05d2d9379e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0b3e6ed4cc6043dab9721e05d2d9379e 2024-11-11T03:41:05,408 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0b3e6ed4cc6043dab9721e05d2d9379e, entries=150, sequenceid=282, filesize=12.0 K 2024-11-11T03:41:05,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/9108d4a0696147ac9ecc01dd8a928440 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/9108d4a0696147ac9ecc01dd8a928440 2024-11-11T03:41:05,412 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/9108d4a0696147ac9ecc01dd8a928440, entries=150, sequenceid=282, filesize=12.0 K 2024-11-11T03:41:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/8483a66804d14986899d804a64ea5e08 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/8483a66804d14986899d804a64ea5e08 2024-11-11T03:41:05,417 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/8483a66804d14986899d804a64ea5e08, entries=150, sequenceid=282, filesize=12.0 K 2024-11-11T03:41:05,418 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ae42fca93718bdb92e7acde763490f33 in 1250ms, sequenceid=282, compaction requested=false 2024-11-11T03:41:05,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:05,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:05,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T03:41:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T03:41:05,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T03:41:05,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-11T03:41:05,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4080 sec 2024-11-11T03:41:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:05,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T03:41:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:05,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/62d50f42f52944d5a58ca452ab21013a is 50, key is test_row_0/A:col10/1731296465523/Put/seqid=0 2024-11-11T03:41:05,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742138_1314 (size=14741) 2024-11-11T03:41:05,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296525584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:05,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296525692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296525902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:05,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/62d50f42f52944d5a58ca452ab21013a 2024-11-11T03:41:05,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/7a1e6f7c9f384692afe9de0d83e6126f is 50, key is test_row_0/B:col10/1731296465523/Put/seqid=0 2024-11-11T03:41:05,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742139_1315 (size=12301) 2024-11-11T03:41:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T03:41:06,172 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:06,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-11T03:41:06,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T03:41:06,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:06,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:06,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296526211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T03:41:06,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T03:41:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/7a1e6f7c9f384692afe9de0d83e6126f 2024-11-11T03:41:06,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/66c2013045134e32b4ae29c2d2f8fb9a is 50, key is test_row_0/C:col10/1731296465523/Put/seqid=0 2024-11-11T03:41:06,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742140_1316 (size=12301) 2024-11-11T03:41:06,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T03:41:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T03:41:06,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T03:41:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:06,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296526722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:06,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/66c2013045134e32b4ae29c2d2f8fb9a 2024-11-11T03:41:06,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/62d50f42f52944d5a58ca452ab21013a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/62d50f42f52944d5a58ca452ab21013a 2024-11-11T03:41:06,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/62d50f42f52944d5a58ca452ab21013a, entries=200, sequenceid=298, filesize=14.4 K 2024-11-11T03:41:06,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/7a1e6f7c9f384692afe9de0d83e6126f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7a1e6f7c9f384692afe9de0d83e6126f 2024-11-11T03:41:06,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7a1e6f7c9f384692afe9de0d83e6126f, entries=150, sequenceid=298, filesize=12.0 K 2024-11-11T03:41:06,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/66c2013045134e32b4ae29c2d2f8fb9a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/66c2013045134e32b4ae29c2d2f8fb9a 2024-11-11T03:41:06,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/66c2013045134e32b4ae29c2d2f8fb9a, entries=150, sequenceid=298, filesize=12.0 K 2024-11-11T03:41:06,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ae42fca93718bdb92e7acde763490f33 in 1242ms, sequenceid=298, compaction requested=true 2024-11-11T03:41:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:06,767 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:06,767 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:06,767 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:06,767 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:06,768 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:41:06,768 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:41:06,768 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,768 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,768 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/b545b07266bf4fe4b799bd2e39cc3c95, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0b3e6ed4cc6043dab9721e05d2d9379e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/62d50f42f52944d5a58ca452ab21013a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=39.0 K 2024-11-11T03:41:06,768 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/ae09bdc97c6043c7a8ff2fdbb1fe58de, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/9108d4a0696147ac9ecc01dd8a928440, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7a1e6f7c9f384692afe9de0d83e6126f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.6 K 2024-11-11T03:41:06,768 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting b545b07266bf4fe4b799bd2e39cc3c95, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296461717 2024-11-11T03:41:06,768 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ae09bdc97c6043c7a8ff2fdbb1fe58de, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296461717 2024-11-11T03:41:06,768 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b3e6ed4cc6043dab9721e05d2d9379e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731296462963 2024-11-11T03:41:06,768 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 9108d4a0696147ac9ecc01dd8a928440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731296462963 2024-11-11T03:41:06,768 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a1e6f7c9f384692afe9de0d83e6126f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731296465178 2024-11-11T03:41:06,769 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62d50f42f52944d5a58ca452ab21013a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731296465178 2024-11-11T03:41:06,775 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:06,775 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:06,775 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/694014f6f99a42fcb369bc5db7ee6406 is 50, key is test_row_0/A:col10/1731296465523/Put/seqid=0 2024-11-11T03:41:06,775 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/d7f946e84c4f47dcbf83f54530b9bc48 is 50, key is test_row_0/B:col10/1731296465523/Put/seqid=0 2024-11-11T03:41:06,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T03:41:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,784 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T03:41:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:06,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742142_1318 (size=13017) 2024-11-11T03:41:06,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742141_1317 (size=13017) 2024-11-11T03:41:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/1c3d725aa9e148e9ba74da5719a767b7 is 50, key is test_row_0/A:col10/1731296465573/Put/seqid=0 2024-11-11T03:41:06,795 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/694014f6f99a42fcb369bc5db7ee6406 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/694014f6f99a42fcb369bc5db7ee6406 2024-11-11T03:41:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T03:41:06,802 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into 694014f6f99a42fcb369bc5db7ee6406(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:06,802 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:06,802 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=13, startTime=1731296466767; duration=0sec 2024-11-11T03:41:06,802 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:06,802 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:41:06,802 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:06,804 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:06,804 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:41:06,804 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:06,804 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/29ac36434e8a46519595257bd6eb4ea9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/8483a66804d14986899d804a64ea5e08, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/66c2013045134e32b4ae29c2d2f8fb9a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.6 K 2024-11-11T03:41:06,805 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ac36434e8a46519595257bd6eb4ea9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296461717 2024-11-11T03:41:06,805 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8483a66804d14986899d804a64ea5e08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731296462963 2024-11-11T03:41:06,805 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c2013045134e32b4ae29c2d2f8fb9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731296465178 2024-11-11T03:41:06,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742143_1319 (size=12301) 2024-11-11T03:41:06,819 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/1c3d725aa9e148e9ba74da5719a767b7 2024-11-11T03:41:06,823 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:06,823 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/c9f4371dc90a4fe9adb7a351984cb505 is 50, key is test_row_0/C:col10/1731296465523/Put/seqid=0 2024-11-11T03:41:06,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5cf7a0b218a94cfda87a14e78acc14ed is 50, key is test_row_0/B:col10/1731296465573/Put/seqid=0 2024-11-11T03:41:06,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742145_1321 (size=13017) 2024-11-11T03:41:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742144_1320 (size=12301) 2024-11-11T03:41:07,190 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/d7f946e84c4f47dcbf83f54530b9bc48 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d7f946e84c4f47dcbf83f54530b9bc48 2024-11-11T03:41:07,194 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into d7f946e84c4f47dcbf83f54530b9bc48(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:07,194 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:07,194 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=13, startTime=1731296466767; duration=0sec 2024-11-11T03:41:07,194 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:07,194 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:41:07,275 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/c9f4371dc90a4fe9adb7a351984cb505 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c9f4371dc90a4fe9adb7a351984cb505 2024-11-11T03:41:07,277 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5cf7a0b218a94cfda87a14e78acc14ed 2024-11-11T03:41:07,279 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into c9f4371dc90a4fe9adb7a351984cb505(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:07,279 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:07,279 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=13, startTime=1731296466767; duration=0sec 2024-11-11T03:41:07,280 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:07,280 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:41:07,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/acf8f2f8f515439fa437622de3fdeb7a is 50, key is test_row_0/C:col10/1731296465573/Put/seqid=0 2024-11-11T03:41:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T03:41:07,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742146_1322 (size=12301) 2024-11-11T03:41:07,317 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/acf8f2f8f515439fa437622de3fdeb7a 2024-11-11T03:41:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/1c3d725aa9e148e9ba74da5719a767b7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1c3d725aa9e148e9ba74da5719a767b7 2024-11-11T03:41:07,328 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1c3d725aa9e148e9ba74da5719a767b7, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T03:41:07,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/5cf7a0b218a94cfda87a14e78acc14ed as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5cf7a0b218a94cfda87a14e78acc14ed 2024-11-11T03:41:07,334 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5cf7a0b218a94cfda87a14e78acc14ed, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T03:41:07,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/acf8f2f8f515439fa437622de3fdeb7a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/acf8f2f8f515439fa437622de3fdeb7a 2024-11-11T03:41:07,339 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/acf8f2f8f515439fa437622de3fdeb7a, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T03:41:07,340 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ae42fca93718bdb92e7acde763490f33 in 556ms, sequenceid=316, compaction requested=false 2024-11-11T03:41:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-11T03:41:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-11T03:41:07,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T03:41:07,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1660 sec 2024-11-11T03:41:07,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.1710 sec 2024-11-11T03:41:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:07,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:41:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:07,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/f3001f6136c14ce985187065008a6ce7 is 50, key is test_row_0/A:col10/1731296467743/Put/seqid=0 2024-11-11T03:41:07,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742147_1323 (size=14741) 2024-11-11T03:41:07,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/f3001f6136c14ce985187065008a6ce7 2024-11-11T03:41:07,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/181b4849203947ffbd66d87112f6ce4a is 50, key is test_row_0/B:col10/1731296467743/Put/seqid=0 2024-11-11T03:41:07,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742148_1324 (size=12301) 2024-11-11T03:41:07,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296467806, max timestamp=1731296467835Num uniques 859; 2024-11-11T03:41:07,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:41:07,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296467806, max timestamp=1731296467835Num uniques 859; 2024-11-11T03:41:07,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:41:07,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296467806, max timestamp=1731296467835Num uniques 859; 2024-11-11T03:41:07,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:41:07,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:07,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296527918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:07,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:07,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:07,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53242 deadline: 1731296528032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:08,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296528032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,036 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53258 deadline: 1731296528062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:08,066 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53240 deadline: 1731296528081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:08,084 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53254 deadline: 1731296528091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:08,096 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in ae42fca93718bdb92e7acde763490f33 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/181b4849203947ffbd66d87112f6ce4a 2024-11-11T03:41:08,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/bb7aee6d8ea84ab89446c3b9551dfcb0 is 50, key is test_row_0/C:col10/1731296467743/Put/seqid=0 2024-11-11T03:41:08,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742149_1325 (size=12301) 2024-11-11T03:41:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296528242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T03:41:08,332 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:08,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-11T03:41:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T03:41:08,334 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:08,334 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:08,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T03:41:08,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T03:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:08,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:08,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:08,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296528551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:08,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/bb7aee6d8ea84ab89446c3b9551dfcb0 2024-11-11T03:41:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/f3001f6136c14ce985187065008a6ce7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f3001f6136c14ce985187065008a6ce7 2024-11-11T03:41:08,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f3001f6136c14ce985187065008a6ce7, entries=200, sequenceid=329, filesize=14.4 K 2024-11-11T03:41:08,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/181b4849203947ffbd66d87112f6ce4a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/181b4849203947ffbd66d87112f6ce4a 2024-11-11T03:41:08,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/181b4849203947ffbd66d87112f6ce4a, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T03:41:08,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/bb7aee6d8ea84ab89446c3b9551dfcb0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bb7aee6d8ea84ab89446c3b9551dfcb0 2024-11-11T03:41:08,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bb7aee6d8ea84ab89446c3b9551dfcb0, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T03:41:08,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ae42fca93718bdb92e7acde763490f33 in 865ms, sequenceid=329, compaction requested=true 2024-11-11T03:41:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:08,612 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:08,612 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:08,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:08,613 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:08,613 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:41:08,613 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:08,613 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d7f946e84c4f47dcbf83f54530b9bc48, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5cf7a0b218a94cfda87a14e78acc14ed, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/181b4849203947ffbd66d87112f6ce4a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.7 K 2024-11-11T03:41:08,613 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:08,613 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:41:08,613 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:08,613 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/694014f6f99a42fcb369bc5db7ee6406, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1c3d725aa9e148e9ba74da5719a767b7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f3001f6136c14ce985187065008a6ce7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=39.1 K 2024-11-11T03:41:08,614 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f946e84c4f47dcbf83f54530b9bc48, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731296465178 2024-11-11T03:41:08,614 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 694014f6f99a42fcb369bc5db7ee6406, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731296465178 2024-11-11T03:41:08,614 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf7a0b218a94cfda87a14e78acc14ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731296465573 2024-11-11T03:41:08,614 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3d725aa9e148e9ba74da5719a767b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731296465573 2024-11-11T03:41:08,614 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 181b4849203947ffbd66d87112f6ce4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296467737 2024-11-11T03:41:08,614 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3001f6136c14ce985187065008a6ce7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296467734 2024-11-11T03:41:08,621 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:08,621 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:08,622 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/3b690383b696456f98aea6c91ffc45bf is 50, key is test_row_0/B:col10/1731296467743/Put/seqid=0 2024-11-11T03:41:08,622 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/a38d54ae7b1048b4977566134311138d is 50, key is test_row_0/A:col10/1731296467743/Put/seqid=0 2024-11-11T03:41:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742150_1326 (size=13119) 2024-11-11T03:41:08,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742151_1327 (size=13119) 2024-11-11T03:41:08,633 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/3b690383b696456f98aea6c91ffc45bf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3b690383b696456f98aea6c91ffc45bf 2024-11-11T03:41:08,638 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into 3b690383b696456f98aea6c91ffc45bf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:08,638 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:08,638 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=13, startTime=1731296468612; duration=0sec 2024-11-11T03:41:08,638 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:08,638 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:41:08,638 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:08,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T03:41:08,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:08,639 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/a38d54ae7b1048b4977566134311138d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/a38d54ae7b1048b4977566134311138d 2024-11-11T03:41:08,639 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T03:41:08,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:08,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:08,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:08,641 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:08,641 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:41:08,641 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:08,641 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c9f4371dc90a4fe9adb7a351984cb505, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/acf8f2f8f515439fa437622de3fdeb7a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bb7aee6d8ea84ab89446c3b9551dfcb0] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.7 K 2024-11-11T03:41:08,642 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f4371dc90a4fe9adb7a351984cb505, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731296465178 2024-11-11T03:41:08,643 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting acf8f2f8f515439fa437622de3fdeb7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731296465573 2024-11-11T03:41:08,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/808ae8dffc024f11bbb54c657932533e is 50, key is test_row_0/A:col10/1731296467916/Put/seqid=0 2024-11-11T03:41:08,646 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting bb7aee6d8ea84ab89446c3b9551dfcb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296467737 2024-11-11T03:41:08,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742152_1328 (size=12301) 2024-11-11T03:41:08,649 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/808ae8dffc024f11bbb54c657932533e 2024-11-11T03:41:08,650 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into a38d54ae7b1048b4977566134311138d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:08,650 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:08,650 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=13, startTime=1731296468612; duration=0sec 2024-11-11T03:41:08,650 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:08,651 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:41:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T03:41:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/3a9f1c749ddb4491872092dfb4bfc24a is 50, key is test_row_0/B:col10/1731296467916/Put/seqid=0 2024-11-11T03:41:08,657 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:08,658 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/dca402a9c53247af84f7023e09929ad5 is 50, key is test_row_0/C:col10/1731296467743/Put/seqid=0 2024-11-11T03:41:08,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742153_1329 (size=12301) 2024-11-11T03:41:08,662 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/3a9f1c749ddb4491872092dfb4bfc24a 2024-11-11T03:41:08,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742154_1330 (size=13119) 2024-11-11T03:41:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/3a424eb123d34e7299bc9198d2040437 is 50, key is test_row_0/C:col10/1731296467916/Put/seqid=0 2024-11-11T03:41:08,672 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/dca402a9c53247af84f7023e09929ad5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/dca402a9c53247af84f7023e09929ad5 2024-11-11T03:41:08,676 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into dca402a9c53247af84f7023e09929ad5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:08,676 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:08,676 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=13, startTime=1731296468612; duration=0sec 2024-11-11T03:41:08,676 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:08,676 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:41:08,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742155_1331 (size=12301) 2024-11-11T03:41:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T03:41:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. as already flushing 2024-11-11T03:41:09,080 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/3a424eb123d34e7299bc9198d2040437 2024-11-11T03:41:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/808ae8dffc024f11bbb54c657932533e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/808ae8dffc024f11bbb54c657932533e 2024-11-11T03:41:09,087 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/808ae8dffc024f11bbb54c657932533e, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T03:41:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/3a9f1c749ddb4491872092dfb4bfc24a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3a9f1c749ddb4491872092dfb4bfc24a 2024-11-11T03:41:09,091 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3a9f1c749ddb4491872092dfb4bfc24a, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T03:41:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/3a424eb123d34e7299bc9198d2040437 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/3a424eb123d34e7299bc9198d2040437 2024-11-11T03:41:09,096 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/3a424eb123d34e7299bc9198d2040437, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T03:41:09,097 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ae42fca93718bdb92e7acde763490f33 in 458ms, sequenceid=367, compaction requested=false 2024-11-11T03:41:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-11T03:41:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-11T03:41:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:09,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T03:41:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:09,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T03:41:09,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 764 msec 2024-11-11T03:41:09,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 768 msec 2024-11-11T03:41:09,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/c9a201c5c38c4f08bc8aeff66966cd6a is 50, key is test_row_0/A:col10/1731296469098/Put/seqid=0 2024-11-11T03:41:09,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742156_1332 (size=14741) 2024-11-11T03:41:09,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296469144, max timestamp=1731296469178Num uniques 859; 2024-11-11T03:41:09,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:41:09,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296469144, max timestamp=1731296469178Num uniques 859; 2024-11-11T03:41:09,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:41:09,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296469144, max timestamp=1731296469178Num uniques 859; 2024-11-11T03:41:09,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:41:09,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296529253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:09,275 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-11-11T03:41:09,275 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:09,275 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,275 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:09,278 INFO [Thread-1187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1187. 2024-11-11T03:41:09,278 DEBUG [Thread-1187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:09,278 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,278 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:09,279 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-11-11T03:41:09,279 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:09,279 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,280 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:09,280 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-11-11T03:41:09,280 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:09,280 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,280 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:09,283 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-11T03:41:09,283 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:09,283 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,283 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:09,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:09,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296529362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:09,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T03:41:09,473 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:09,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/c9a201c5c38c4f08bc8aeff66966cd6a 2024-11-11T03:41:09,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/38ab6545761145eca4e42ea8356abaa1 is 50, key is test_row_0/B:col10/1731296469098/Put/seqid=0 2024-11-11T03:41:09,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742157_1333 (size=12301) 2024-11-11T03:41:09,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296529573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:09,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1731296529883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:09,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/38ab6545761145eca4e42ea8356abaa1 2024-11-11T03:41:09,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/7d755c368f6749b78f6bd9294763544c is 50, key is test_row_0/C:col10/1731296469098/Put/seqid=0 2024-11-11T03:41:09,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742158_1334 (size=12301) 2024-11-11T03:41:10,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/7d755c368f6749b78f6bd9294763544c 2024-11-11T03:41:10,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/c9a201c5c38c4f08bc8aeff66966cd6a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c9a201c5c38c4f08bc8aeff66966cd6a 2024-11-11T03:41:10,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c9a201c5c38c4f08bc8aeff66966cd6a, entries=200, sequenceid=380, filesize=14.4 K 2024-11-11T03:41:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/38ab6545761145eca4e42ea8356abaa1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/38ab6545761145eca4e42ea8356abaa1 2024-11-11T03:41:10,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/38ab6545761145eca4e42ea8356abaa1, entries=150, sequenceid=380, filesize=12.0 K 2024-11-11T03:41:10,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/7d755c368f6749b78f6bd9294763544c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/7d755c368f6749b78f6bd9294763544c 2024-11-11T03:41:10,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/7d755c368f6749b78f6bd9294763544c, entries=150, sequenceid=380, filesize=12.0 K 2024-11-11T03:41:10,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ae42fca93718bdb92e7acde763490f33 in 1278ms, sequenceid=380, compaction requested=true 2024-11-11T03:41:10,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:10,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:10,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:10,378 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:10,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:10,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:10,378 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:10,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae42fca93718bdb92e7acde763490f33:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:10,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:10,379 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:10,379 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/A is initiating minor compaction (all files) 2024-11-11T03:41:10,379 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:10,379 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/A in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:10,379 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/B is initiating minor compaction (all files) 2024-11-11T03:41:10,379 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/a38d54ae7b1048b4977566134311138d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/808ae8dffc024f11bbb54c657932533e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c9a201c5c38c4f08bc8aeff66966cd6a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=39.2 K 2024-11-11T03:41:10,379 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/B in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:10,380 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3b690383b696456f98aea6c91ffc45bf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3a9f1c749ddb4491872092dfb4bfc24a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/38ab6545761145eca4e42ea8356abaa1] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.8 K 2024-11-11T03:41:10,380 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a38d54ae7b1048b4977566134311138d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296467737 2024-11-11T03:41:10,380 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b690383b696456f98aea6c91ffc45bf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296467737 2024-11-11T03:41:10,380 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 808ae8dffc024f11bbb54c657932533e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731296467888 2024-11-11T03:41:10,380 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a9f1c749ddb4491872092dfb4bfc24a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731296467888 2024-11-11T03:41:10,381 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a201c5c38c4f08bc8aeff66966cd6a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731296469083 2024-11-11T03:41:10,381 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ab6545761145eca4e42ea8356abaa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731296469089 2024-11-11T03:41:10,388 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#B#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:10,388 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#A#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:10,388 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/2a61f4de9f8c45c1b12b2efdc32ca8d2 is 50, key is test_row_0/B:col10/1731296469098/Put/seqid=0 2024-11-11T03:41:10,388 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/469ee7c89dd0424194fc572beff8db0c is 50, key is test_row_0/A:col10/1731296469098/Put/seqid=0 2024-11-11T03:41:10,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742159_1335 (size=13221) 2024-11-11T03:41:10,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742160_1336 (size=13221) 2024-11-11T03:41:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:10,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T03:41:10,393 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-11T03:41:10,393 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:10,393 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:10,393 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:10,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:10,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:10,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/cf0beeb9c6714cdb9a52d65abe8dffdf is 50, key is test_row_0/A:col10/1731296469234/Put/seqid=0 2024-11-11T03:41:10,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742161_1337 (size=12301) 2024-11-11T03:41:10,799 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/2a61f4de9f8c45c1b12b2efdc32ca8d2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/2a61f4de9f8c45c1b12b2efdc32ca8d2 2024-11-11T03:41:10,799 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/469ee7c89dd0424194fc572beff8db0c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/469ee7c89dd0424194fc572beff8db0c 2024-11-11T03:41:10,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/cf0beeb9c6714cdb9a52d65abe8dffdf 2024-11-11T03:41:10,806 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/A of ae42fca93718bdb92e7acde763490f33 into 469ee7c89dd0424194fc572beff8db0c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:10,806 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/B of ae42fca93718bdb92e7acde763490f33 into 2a61f4de9f8c45c1b12b2efdc32ca8d2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:10,806 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:10,806 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:10,806 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/B, priority=13, startTime=1731296470378; duration=0sec 2024-11-11T03:41:10,806 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/A, priority=13, startTime=1731296470378; duration=0sec 2024-11-11T03:41:10,806 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:10,806 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:A 2024-11-11T03:41:10,806 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:10,806 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:B 2024-11-11T03:41:10,806 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:10,808 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:10,808 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): ae42fca93718bdb92e7acde763490f33/C is initiating minor compaction (all files) 2024-11-11T03:41:10,808 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae42fca93718bdb92e7acde763490f33/C in TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:10,808 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/dca402a9c53247af84f7023e09929ad5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/3a424eb123d34e7299bc9198d2040437, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/7d755c368f6749b78f6bd9294763544c] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp, totalSize=36.8 K 2024-11-11T03:41:10,808 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca402a9c53247af84f7023e09929ad5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731296467737 2024-11-11T03:41:10,809 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a424eb123d34e7299bc9198d2040437, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731296467888 2024-11-11T03:41:10,809 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d755c368f6749b78f6bd9294763544c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731296469089 2024-11-11T03:41:10,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/7e742ff8790d4a4aa4d27198f2821c28 is 50, key is test_row_0/B:col10/1731296469234/Put/seqid=0 2024-11-11T03:41:10,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742162_1338 (size=12301) 2024-11-11T03:41:10,816 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae42fca93718bdb92e7acde763490f33#C#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:10,816 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/b23d806e58604eddb8b07e665d305000 is 50, key is test_row_0/C:col10/1731296469098/Put/seqid=0 2024-11-11T03:41:10,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742163_1339 (size=13221) 2024-11-11T03:41:11,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/7e742ff8790d4a4aa4d27198f2821c28 2024-11-11T03:41:11,227 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/b23d806e58604eddb8b07e665d305000 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/b23d806e58604eddb8b07e665d305000 2024-11-11T03:41:11,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/45967082dc4e40c6a2864d8ad6a8a4ee is 50, key is test_row_0/C:col10/1731296469234/Put/seqid=0 2024-11-11T03:41:11,232 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae42fca93718bdb92e7acde763490f33/C of ae42fca93718bdb92e7acde763490f33 into b23d806e58604eddb8b07e665d305000(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:11,232 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:11,232 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., storeName=ae42fca93718bdb92e7acde763490f33/C, priority=13, startTime=1731296470378; duration=0sec 2024-11-11T03:41:11,232 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:11,232 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae42fca93718bdb92e7acde763490f33:C 2024-11-11T03:41:11,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742164_1340 (size=12301) 2024-11-11T03:41:11,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/45967082dc4e40c6a2864d8ad6a8a4ee 2024-11-11T03:41:11,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/cf0beeb9c6714cdb9a52d65abe8dffdf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/cf0beeb9c6714cdb9a52d65abe8dffdf 2024-11-11T03:41:11,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/cf0beeb9c6714cdb9a52d65abe8dffdf, entries=150, sequenceid=417, filesize=12.0 K 2024-11-11T03:41:11,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/7e742ff8790d4a4aa4d27198f2821c28 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7e742ff8790d4a4aa4d27198f2821c28 2024-11-11T03:41:11,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7e742ff8790d4a4aa4d27198f2821c28, entries=150, sequenceid=417, filesize=12.0 K 2024-11-11T03:41:11,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/45967082dc4e40c6a2864d8ad6a8a4ee as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/45967082dc4e40c6a2864d8ad6a8a4ee 2024-11-11T03:41:11,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/45967082dc4e40c6a2864d8ad6a8a4ee, entries=150, sequenceid=417, filesize=12.0 K 2024-11-11T03:41:11,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for ae42fca93718bdb92e7acde763490f33 in 1256ms, sequenceid=417, compaction requested=false 2024-11-11T03:41:11,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae42fca93718bdb92e7acde763490f33: 2024-11-11T03:41:12,621 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:41:18,106 INFO [Thread-1178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1178. 2024-11-11T03:41:18,106 DEBUG [Thread-1178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:18,106 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,106 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:18,114 INFO [Thread-1176 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1176. 2024-11-11T03:41:18,115 DEBUG [Thread-1176 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:18,115 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,115 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:18,124 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-11T03:41:18,124 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:18,124 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,124 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:18,143 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-11T03:41:18,143 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:18,143 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,143 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:18,143 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T03:41:18,143 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3590 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3487 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3518 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3592 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3491 2024-11-11T03:41:18,144 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T03:41:18,144 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:41:18,144 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:41:18,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:18,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:18,145 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T03:41:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T03:41:18,149 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296478148"}]},"ts":"1731296478148"} 2024-11-11T03:41:18,150 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T03:41:18,150 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T03:41:18,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:41:18,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, UNASSIGN}] 2024-11-11T03:41:18,154 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, UNASSIGN 2024-11-11T03:41:18,155 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ae42fca93718bdb92e7acde763490f33, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:18,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, UNASSIGN because future has completed 2024-11-11T03:41:18,156 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:41:18,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:41:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T03:41:18,310 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:18,310 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:41:18,310 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing ae42fca93718bdb92e7acde763490f33, disabling compactions & flushes 2024-11-11T03:41:18,310 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:18,310 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:18,310 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. after waiting 0 ms 2024-11-11T03:41:18,310 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:18,310 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing ae42fca93718bdb92e7acde763490f33 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T03:41:18,311 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=A 2024-11-11T03:41:18,311 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:18,311 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=B 2024-11-11T03:41:18,311 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:18,311 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae42fca93718bdb92e7acde763490f33, store=C 2024-11-11T03:41:18,311 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:18,372 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0c23b49a654148b69ba6f950363904a3 is 50, key is test_row_1/A:col10/1731296478112/Put/seqid=0 2024-11-11T03:41:18,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742165_1341 (size=9857) 2024-11-11T03:41:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T03:41:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T03:41:18,781 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0c23b49a654148b69ba6f950363904a3 2024-11-11T03:41:18,797 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/f57b895f90db4f2eada25b244fd9e5d3 is 50, key is test_row_1/B:col10/1731296478112/Put/seqid=0 2024-11-11T03:41:18,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742166_1342 (size=9857) 2024-11-11T03:41:19,202 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/f57b895f90db4f2eada25b244fd9e5d3 2024-11-11T03:41:19,217 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/08bd726cf8ed4bf799c3566b7e583f56 is 50, key is test_row_1/C:col10/1731296478112/Put/seqid=0 2024-11-11T03:41:19,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742167_1343 (size=9857) 2024-11-11T03:41:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T03:41:19,622 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/08bd726cf8ed4bf799c3566b7e583f56 2024-11-11T03:41:19,632 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/A/0c23b49a654148b69ba6f950363904a3 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0c23b49a654148b69ba6f950363904a3 2024-11-11T03:41:19,637 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0c23b49a654148b69ba6f950363904a3, entries=100, sequenceid=427, filesize=9.6 K 2024-11-11T03:41:19,638 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/B/f57b895f90db4f2eada25b244fd9e5d3 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f57b895f90db4f2eada25b244fd9e5d3 2024-11-11T03:41:19,642 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f57b895f90db4f2eada25b244fd9e5d3, entries=100, sequenceid=427, filesize=9.6 K 2024-11-11T03:41:19,643 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/.tmp/C/08bd726cf8ed4bf799c3566b7e583f56 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/08bd726cf8ed4bf799c3566b7e583f56 2024-11-11T03:41:19,648 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/08bd726cf8ed4bf799c3566b7e583f56, entries=100, sequenceid=427, filesize=9.6 K 2024-11-11T03:41:19,649 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ae42fca93718bdb92e7acde763490f33 in 1339ms, sequenceid=427, compaction requested=true 2024-11-11T03:41:19,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/6a2824560d4440c5b0df4f868faa62a4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0fcdbadc8f8248b589356f6b0b8d6383, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f5a6835da2514c6c8653e85a9fa1fd69, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8e8793156f21432288f02c037d46bb7e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/3f2ce7c72af542c19c80447d9e1825f0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/e66e0842732b49229c0a803a7a9ef112, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2bd3746ddf2d41638a803e69af9cae61, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0ba5e6f15d1845f0ba428f00dbfa1509, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2b45282ef34343b4963f1bdc82137c1d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c93a6c6b9a194336beb7aee9e3d507b0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1eac1e4c7f5d47759d2095a843422251, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/776021643eda45cfa56f0aa3abf023c6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8148c6d9a00b4af089b2dbc1d340a8f2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/ac52be863b694de6849417a1f2bc269b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/73e749016e134debb6b1621dcf08643d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/33046a34f51d47df89d498b6bcfb5a4e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/de99ee13896a435f83a33971ea81e49e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/070d9da847f04dcaa257523d1346699d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/b545b07266bf4fe4b799bd2e39cc3c95, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0b3e6ed4cc6043dab9721e05d2d9379e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/62d50f42f52944d5a58ca452ab21013a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/694014f6f99a42fcb369bc5db7ee6406, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1c3d725aa9e148e9ba74da5719a767b7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f3001f6136c14ce985187065008a6ce7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/a38d54ae7b1048b4977566134311138d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/808ae8dffc024f11bbb54c657932533e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c9a201c5c38c4f08bc8aeff66966cd6a] to archive 2024-11-11T03:41:19,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:41:19,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/6a2824560d4440c5b0df4f868faa62a4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/6a2824560d4440c5b0df4f868faa62a4 2024-11-11T03:41:19,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0fcdbadc8f8248b589356f6b0b8d6383 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0fcdbadc8f8248b589356f6b0b8d6383 2024-11-11T03:41:19,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f5a6835da2514c6c8653e85a9fa1fd69 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f5a6835da2514c6c8653e85a9fa1fd69 2024-11-11T03:41:19,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8e8793156f21432288f02c037d46bb7e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8e8793156f21432288f02c037d46bb7e 2024-11-11T03:41:19,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/3f2ce7c72af542c19c80447d9e1825f0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/3f2ce7c72af542c19c80447d9e1825f0 2024-11-11T03:41:19,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/e66e0842732b49229c0a803a7a9ef112 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/e66e0842732b49229c0a803a7a9ef112 2024-11-11T03:41:19,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2bd3746ddf2d41638a803e69af9cae61 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2bd3746ddf2d41638a803e69af9cae61 2024-11-11T03:41:19,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0ba5e6f15d1845f0ba428f00dbfa1509 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0ba5e6f15d1845f0ba428f00dbfa1509 2024-11-11T03:41:19,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2b45282ef34343b4963f1bdc82137c1d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/2b45282ef34343b4963f1bdc82137c1d 2024-11-11T03:41:19,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c93a6c6b9a194336beb7aee9e3d507b0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c93a6c6b9a194336beb7aee9e3d507b0 2024-11-11T03:41:19,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1eac1e4c7f5d47759d2095a843422251 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1eac1e4c7f5d47759d2095a843422251 2024-11-11T03:41:19,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/776021643eda45cfa56f0aa3abf023c6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/776021643eda45cfa56f0aa3abf023c6 2024-11-11T03:41:19,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8148c6d9a00b4af089b2dbc1d340a8f2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/8148c6d9a00b4af089b2dbc1d340a8f2 2024-11-11T03:41:19,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/ac52be863b694de6849417a1f2bc269b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/ac52be863b694de6849417a1f2bc269b 2024-11-11T03:41:19,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/73e749016e134debb6b1621dcf08643d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/73e749016e134debb6b1621dcf08643d 2024-11-11T03:41:19,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/33046a34f51d47df89d498b6bcfb5a4e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/33046a34f51d47df89d498b6bcfb5a4e 2024-11-11T03:41:19,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/de99ee13896a435f83a33971ea81e49e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/de99ee13896a435f83a33971ea81e49e 2024-11-11T03:41:19,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/070d9da847f04dcaa257523d1346699d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/070d9da847f04dcaa257523d1346699d 2024-11-11T03:41:19,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/b545b07266bf4fe4b799bd2e39cc3c95 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/b545b07266bf4fe4b799bd2e39cc3c95 2024-11-11T03:41:19,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0b3e6ed4cc6043dab9721e05d2d9379e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0b3e6ed4cc6043dab9721e05d2d9379e 2024-11-11T03:41:19,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/62d50f42f52944d5a58ca452ab21013a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/62d50f42f52944d5a58ca452ab21013a 2024-11-11T03:41:19,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/694014f6f99a42fcb369bc5db7ee6406 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/694014f6f99a42fcb369bc5db7ee6406 2024-11-11T03:41:19,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1c3d725aa9e148e9ba74da5719a767b7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/1c3d725aa9e148e9ba74da5719a767b7 2024-11-11T03:41:19,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f3001f6136c14ce985187065008a6ce7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/f3001f6136c14ce985187065008a6ce7 2024-11-11T03:41:19,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/a38d54ae7b1048b4977566134311138d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/a38d54ae7b1048b4977566134311138d 2024-11-11T03:41:19,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/808ae8dffc024f11bbb54c657932533e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/808ae8dffc024f11bbb54c657932533e 2024-11-11T03:41:19,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c9a201c5c38c4f08bc8aeff66966cd6a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/c9a201c5c38c4f08bc8aeff66966cd6a 2024-11-11T03:41:19,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/4d858faf3baa4fb6a70409059ae8fa41, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c4134e780784816a98802368b7b62f4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/fd29aae11c0a46089016bb68b69b3dd2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f19662267c004c96aaf154065c218712, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8716f77849ba40ad94cd4574ec4afb8e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/aee8d745deb64243b3d0288106f79a3c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/1b36d0dd769e4a9d911e97fdb529d736, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/e304293e20d74aa597e4af1702ab084a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/b84e2f60b0174e6594a7b698c7e25943, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c5dad7212f9429f9f2de8d8335da86b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/efd5b46d822c45f9b08621ba7e63003f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8047662ae57840da97d39bf37f0f2c9f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/908af378c40148df973b8ebb00cd6485, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/c30c10bf51ab45bb8e1fd29bb26bbc02, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d0acce98578f4163ae177a4186dd656f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d017b78040da429dbd8bbfa6b14ad119, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/95ee4d8560214e269f7a34201e5efa7b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/ae09bdc97c6043c7a8ff2fdbb1fe58de, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/0c8a2ec0179747baa48b0993d4947636, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/9108d4a0696147ac9ecc01dd8a928440, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d7f946e84c4f47dcbf83f54530b9bc48, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7a1e6f7c9f384692afe9de0d83e6126f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5cf7a0b218a94cfda87a14e78acc14ed, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3b690383b696456f98aea6c91ffc45bf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/181b4849203947ffbd66d87112f6ce4a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3a9f1c749ddb4491872092dfb4bfc24a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/38ab6545761145eca4e42ea8356abaa1] to archive 2024-11-11T03:41:19,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:41:19,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/4d858faf3baa4fb6a70409059ae8fa41 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/4d858faf3baa4fb6a70409059ae8fa41 2024-11-11T03:41:19,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c4134e780784816a98802368b7b62f4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c4134e780784816a98802368b7b62f4 2024-11-11T03:41:19,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/fd29aae11c0a46089016bb68b69b3dd2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/fd29aae11c0a46089016bb68b69b3dd2 2024-11-11T03:41:19,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f19662267c004c96aaf154065c218712 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f19662267c004c96aaf154065c218712 2024-11-11T03:41:19,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8716f77849ba40ad94cd4574ec4afb8e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8716f77849ba40ad94cd4574ec4afb8e 2024-11-11T03:41:19,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/aee8d745deb64243b3d0288106f79a3c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/aee8d745deb64243b3d0288106f79a3c 2024-11-11T03:41:19,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/1b36d0dd769e4a9d911e97fdb529d736 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/1b36d0dd769e4a9d911e97fdb529d736 2024-11-11T03:41:19,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/e304293e20d74aa597e4af1702ab084a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/e304293e20d74aa597e4af1702ab084a 2024-11-11T03:41:19,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/b84e2f60b0174e6594a7b698c7e25943 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/b84e2f60b0174e6594a7b698c7e25943 2024-11-11T03:41:19,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c5dad7212f9429f9f2de8d8335da86b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5c5dad7212f9429f9f2de8d8335da86b 2024-11-11T03:41:19,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/efd5b46d822c45f9b08621ba7e63003f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/efd5b46d822c45f9b08621ba7e63003f 2024-11-11T03:41:19,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8047662ae57840da97d39bf37f0f2c9f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/8047662ae57840da97d39bf37f0f2c9f 2024-11-11T03:41:19,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/908af378c40148df973b8ebb00cd6485 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/908af378c40148df973b8ebb00cd6485 2024-11-11T03:41:19,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/c30c10bf51ab45bb8e1fd29bb26bbc02 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/c30c10bf51ab45bb8e1fd29bb26bbc02 2024-11-11T03:41:19,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d0acce98578f4163ae177a4186dd656f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d0acce98578f4163ae177a4186dd656f 2024-11-11T03:41:19,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d017b78040da429dbd8bbfa6b14ad119 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d017b78040da429dbd8bbfa6b14ad119 2024-11-11T03:41:19,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/95ee4d8560214e269f7a34201e5efa7b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/95ee4d8560214e269f7a34201e5efa7b 2024-11-11T03:41:19,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/ae09bdc97c6043c7a8ff2fdbb1fe58de to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/ae09bdc97c6043c7a8ff2fdbb1fe58de 2024-11-11T03:41:19,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/0c8a2ec0179747baa48b0993d4947636 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/0c8a2ec0179747baa48b0993d4947636 2024-11-11T03:41:19,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/9108d4a0696147ac9ecc01dd8a928440 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/9108d4a0696147ac9ecc01dd8a928440 2024-11-11T03:41:19,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d7f946e84c4f47dcbf83f54530b9bc48 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/d7f946e84c4f47dcbf83f54530b9bc48 2024-11-11T03:41:19,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7a1e6f7c9f384692afe9de0d83e6126f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7a1e6f7c9f384692afe9de0d83e6126f 2024-11-11T03:41:19,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5cf7a0b218a94cfda87a14e78acc14ed to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/5cf7a0b218a94cfda87a14e78acc14ed 2024-11-11T03:41:19,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3b690383b696456f98aea6c91ffc45bf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3b690383b696456f98aea6c91ffc45bf 2024-11-11T03:41:19,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/181b4849203947ffbd66d87112f6ce4a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/181b4849203947ffbd66d87112f6ce4a 2024-11-11T03:41:19,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3a9f1c749ddb4491872092dfb4bfc24a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/3a9f1c749ddb4491872092dfb4bfc24a 2024-11-11T03:41:19,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/38ab6545761145eca4e42ea8356abaa1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/38ab6545761145eca4e42ea8356abaa1 2024-11-11T03:41:19,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/0da54b8209bd49e18353c945bc96888b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/05aef7a8f7b24ffb98f003996e38a5e7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/cb349945b1e342c88676fbade439d51d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/5f6a7227006d4e8aaa9ce72ba0d902ae, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/09328d304d9542e1b2eb60738dc556a9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/477272de5ad74b589c9e47da5405d843, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ff071ec8d0d645bbab9c198d0bc552eb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/2adb5050e7244c79aa968ea4bf47761b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/9e1e2e0cdb1c41b68f0993b53b67349f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/6240fd2d2ce840758955ffa7672375a0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bdb3df1fe66f46058412be97d14df134, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c5e5c14288234bab85f8bc10fc22e130, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/890ea41fb04044a288829166f7f9a578, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ffe983282aed41a982b2eaa2cb340a6b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/030358e0392c4f5a97e35df1dd04745f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/d1b7d84d81614f61a1aace5ae7d7c5b7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c42a7ce1a4ec4328936ca7a6cf637cbe, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/29ac36434e8a46519595257bd6eb4ea9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/967a228140f745dbb37dea52b57442f0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/8483a66804d14986899d804a64ea5e08, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c9f4371dc90a4fe9adb7a351984cb505, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/66c2013045134e32b4ae29c2d2f8fb9a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/acf8f2f8f515439fa437622de3fdeb7a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/dca402a9c53247af84f7023e09929ad5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bb7aee6d8ea84ab89446c3b9551dfcb0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/3a424eb123d34e7299bc9198d2040437, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/7d755c368f6749b78f6bd9294763544c] to archive 2024-11-11T03:41:19,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:41:19,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/0da54b8209bd49e18353c945bc96888b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/0da54b8209bd49e18353c945bc96888b 2024-11-11T03:41:19,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/05aef7a8f7b24ffb98f003996e38a5e7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/05aef7a8f7b24ffb98f003996e38a5e7 2024-11-11T03:41:19,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/cb349945b1e342c88676fbade439d51d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/cb349945b1e342c88676fbade439d51d 2024-11-11T03:41:19,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/5f6a7227006d4e8aaa9ce72ba0d902ae to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/5f6a7227006d4e8aaa9ce72ba0d902ae 2024-11-11T03:41:19,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/09328d304d9542e1b2eb60738dc556a9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/09328d304d9542e1b2eb60738dc556a9 2024-11-11T03:41:19,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/477272de5ad74b589c9e47da5405d843 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/477272de5ad74b589c9e47da5405d843 2024-11-11T03:41:19,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ff071ec8d0d645bbab9c198d0bc552eb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ff071ec8d0d645bbab9c198d0bc552eb 2024-11-11T03:41:19,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/2adb5050e7244c79aa968ea4bf47761b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/2adb5050e7244c79aa968ea4bf47761b 2024-11-11T03:41:19,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/9e1e2e0cdb1c41b68f0993b53b67349f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/9e1e2e0cdb1c41b68f0993b53b67349f 2024-11-11T03:41:19,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/6240fd2d2ce840758955ffa7672375a0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/6240fd2d2ce840758955ffa7672375a0 2024-11-11T03:41:19,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bdb3df1fe66f46058412be97d14df134 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bdb3df1fe66f46058412be97d14df134 2024-11-11T03:41:19,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c5e5c14288234bab85f8bc10fc22e130 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c5e5c14288234bab85f8bc10fc22e130 2024-11-11T03:41:19,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/890ea41fb04044a288829166f7f9a578 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/890ea41fb04044a288829166f7f9a578 2024-11-11T03:41:19,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ffe983282aed41a982b2eaa2cb340a6b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/ffe983282aed41a982b2eaa2cb340a6b 2024-11-11T03:41:19,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/030358e0392c4f5a97e35df1dd04745f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/030358e0392c4f5a97e35df1dd04745f 2024-11-11T03:41:19,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/d1b7d84d81614f61a1aace5ae7d7c5b7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/d1b7d84d81614f61a1aace5ae7d7c5b7 2024-11-11T03:41:19,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c42a7ce1a4ec4328936ca7a6cf637cbe to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c42a7ce1a4ec4328936ca7a6cf637cbe 2024-11-11T03:41:19,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/29ac36434e8a46519595257bd6eb4ea9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/29ac36434e8a46519595257bd6eb4ea9 2024-11-11T03:41:19,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/967a228140f745dbb37dea52b57442f0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/967a228140f745dbb37dea52b57442f0 2024-11-11T03:41:19,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/8483a66804d14986899d804a64ea5e08 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/8483a66804d14986899d804a64ea5e08 2024-11-11T03:41:19,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c9f4371dc90a4fe9adb7a351984cb505 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/c9f4371dc90a4fe9adb7a351984cb505 2024-11-11T03:41:19,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/66c2013045134e32b4ae29c2d2f8fb9a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/66c2013045134e32b4ae29c2d2f8fb9a 2024-11-11T03:41:19,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/acf8f2f8f515439fa437622de3fdeb7a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/acf8f2f8f515439fa437622de3fdeb7a 2024-11-11T03:41:19,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/dca402a9c53247af84f7023e09929ad5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/dca402a9c53247af84f7023e09929ad5 2024-11-11T03:41:19,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bb7aee6d8ea84ab89446c3b9551dfcb0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/bb7aee6d8ea84ab89446c3b9551dfcb0 2024-11-11T03:41:19,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/3a424eb123d34e7299bc9198d2040437 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/3a424eb123d34e7299bc9198d2040437 2024-11-11T03:41:19,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/7d755c368f6749b78f6bd9294763544c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/7d755c368f6749b78f6bd9294763544c 2024-11-11T03:41:19,759 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/recovered.edits/430.seqid, newMaxSeqId=430, maxSeqId=1 2024-11-11T03:41:19,759 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. 2024-11-11T03:41:19,759 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for ae42fca93718bdb92e7acde763490f33: Waiting for close lock at 1731296478310Running coprocessor pre-close hooks at 1731296478310Disabling compacts and flushes for region at 1731296478310Disabling writes for close at 1731296478310Obtaining lock to block concurrent updates at 1731296478311 (+1 ms)Preparing flush snapshotting stores in ae42fca93718bdb92e7acde763490f33 at 1731296478311Finished memstore snapshotting TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731296478311Flushing stores of TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33. at 1731296478364 (+53 ms)Flushing ae42fca93718bdb92e7acde763490f33/A: creating writer at 1731296478364Flushing ae42fca93718bdb92e7acde763490f33/A: appending metadata at 1731296478371 (+7 ms)Flushing ae42fca93718bdb92e7acde763490f33/A: closing flushed file at 1731296478371Flushing ae42fca93718bdb92e7acde763490f33/B: creating writer at 1731296478791 (+420 ms)Flushing ae42fca93718bdb92e7acde763490f33/B: appending metadata at 1731296478796 (+5 ms)Flushing ae42fca93718bdb92e7acde763490f33/B: closing flushed file at 1731296478796Flushing ae42fca93718bdb92e7acde763490f33/C: creating writer at 1731296479213 (+417 ms)Flushing ae42fca93718bdb92e7acde763490f33/C: appending metadata at 1731296479216 (+3 ms)Flushing ae42fca93718bdb92e7acde763490f33/C: closing flushed file at 1731296479216Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38d0e2bc: reopening flushed file at 1731296479631 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f4f7f68: reopening flushed file at 1731296479637 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cb1678: reopening flushed file at 1731296479642 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ae42fca93718bdb92e7acde763490f33 in 1339ms, sequenceid=427, compaction requested=true at 1731296479649 (+7 ms)Writing region close event to WAL at 1731296479756 (+107 ms)Running coprocessor post-close hooks at 1731296479759 (+3 ms)Closed at 1731296479759 2024-11-11T03:41:19,761 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:19,761 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ae42fca93718bdb92e7acde763490f33, regionState=CLOSED 2024-11-11T03:41:19,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:41:19,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T03:41:19,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure ae42fca93718bdb92e7acde763490f33, server=3ffe7f9d089e,34473,1731296384288 in 1.6060 sec 2024-11-11T03:41:19,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T03:41:19,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae42fca93718bdb92e7acde763490f33, UNASSIGN in 1.6110 sec 2024-11-11T03:41:19,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-11T03:41:19,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6150 sec 2024-11-11T03:41:19,768 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296479768"}]},"ts":"1731296479768"} 2024-11-11T03:41:19,769 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T03:41:19,769 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T03:41:19,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6250 sec 2024-11-11T03:41:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T03:41:20,292 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:20,293 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T03:41:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:20,295 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T03:41:20,297 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:20,299 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:20,303 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/recovered.edits] 2024-11-11T03:41:20,309 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0c23b49a654148b69ba6f950363904a3 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/0c23b49a654148b69ba6f950363904a3 2024-11-11T03:41:20,311 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/469ee7c89dd0424194fc572beff8db0c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/469ee7c89dd0424194fc572beff8db0c 2024-11-11T03:41:20,313 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/cf0beeb9c6714cdb9a52d65abe8dffdf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/A/cf0beeb9c6714cdb9a52d65abe8dffdf 2024-11-11T03:41:20,316 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/2a61f4de9f8c45c1b12b2efdc32ca8d2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/2a61f4de9f8c45c1b12b2efdc32ca8d2 2024-11-11T03:41:20,317 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7e742ff8790d4a4aa4d27198f2821c28 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/7e742ff8790d4a4aa4d27198f2821c28 2024-11-11T03:41:20,318 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f57b895f90db4f2eada25b244fd9e5d3 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/B/f57b895f90db4f2eada25b244fd9e5d3 2024-11-11T03:41:20,321 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/08bd726cf8ed4bf799c3566b7e583f56 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/08bd726cf8ed4bf799c3566b7e583f56 2024-11-11T03:41:20,322 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/45967082dc4e40c6a2864d8ad6a8a4ee to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/45967082dc4e40c6a2864d8ad6a8a4ee 2024-11-11T03:41:20,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/b23d806e58604eddb8b07e665d305000 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/C/b23d806e58604eddb8b07e665d305000 2024-11-11T03:41:20,326 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/recovered.edits/430.seqid to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33/recovered.edits/430.seqid 2024-11-11T03:41:20,326 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/ae42fca93718bdb92e7acde763490f33 2024-11-11T03:41:20,326 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T03:41:20,328 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:20,330 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T03:41:20,332 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T03:41:20,333 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:20,333 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T03:41:20,333 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731296480333"}]},"ts":"9223372036854775807"} 2024-11-11T03:41:20,334 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T03:41:20,334 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae42fca93718bdb92e7acde763490f33, NAME => 'TestAcidGuarantees,,1731296448076.ae42fca93718bdb92e7acde763490f33.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T03:41:20,334 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T03:41:20,335 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731296480335"}]},"ts":"9223372036854775807"} 2024-11-11T03:41:20,336 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T03:41:20,337 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:20,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-11T03:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T03:41:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T03:41:20,402 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:20,412 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 358), ProcessCount=11 (was 11), AvailableMemoryMB=11830 (was 11967) 2024-11-11T03:41:20,420 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=11830 2024-11-11T03:41:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:41:20,421 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:41:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:20,422 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T03:41:20,423 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:20,423 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-11T03:41:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T03:41:20,423 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T03:41:20,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742168_1344 (size=1061) 2024-11-11T03:41:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T03:41:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T03:41:20,834 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:41:20,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742169_1345 (size=53) 2024-11-11T03:41:20,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:41:20,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4216aa280844bc244aeea907e086297f, disabling compactions & flushes 2024-11-11T03:41:20,847 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:20,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:20,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. after waiting 0 ms 2024-11-11T03:41:20,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:20,847 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:20,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4216aa280844bc244aeea907e086297f: Waiting for close lock at 1731296480847Disabling compacts and flushes for region at 1731296480847Disabling writes for close at 1731296480847Writing region close event to WAL at 1731296480847Closed at 1731296480847 2024-11-11T03:41:20,849 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T03:41:20,849 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731296480849"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731296480849"}]},"ts":"1731296480849"} 2024-11-11T03:41:20,851 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T03:41:20,852 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T03:41:20,852 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296480852"}]},"ts":"1731296480852"} 2024-11-11T03:41:20,853 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T03:41:20,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, ASSIGN}] 2024-11-11T03:41:20,854 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, ASSIGN 2024-11-11T03:41:20,854 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, ASSIGN; state=OFFLINE, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=false 2024-11-11T03:41:21,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:21,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, ASSIGN because future has completed 2024-11-11T03:41:21,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:41:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T03:41:21,172 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:21,173 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:41:21,174 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,174 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:41:21,174 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,174 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,178 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,180 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:21,180 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4216aa280844bc244aeea907e086297f columnFamilyName A 2024-11-11T03:41:21,180 DEBUG [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:21,181 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(327): Store=4216aa280844bc244aeea907e086297f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:21,181 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,183 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:21,183 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4216aa280844bc244aeea907e086297f columnFamilyName B 2024-11-11T03:41:21,183 DEBUG [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:21,184 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(327): Store=4216aa280844bc244aeea907e086297f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:21,184 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,186 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:21,186 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4216aa280844bc244aeea907e086297f columnFamilyName C 2024-11-11T03:41:21,186 DEBUG [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:21,187 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(327): Store=4216aa280844bc244aeea907e086297f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:21,187 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:21,187 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,188 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,189 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,190 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,190 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,191 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:41:21,193 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,196 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:41:21,197 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 4216aa280844bc244aeea907e086297f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69201044, jitterRate=0.03117591142654419}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:41:21,197 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:21,198 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 4216aa280844bc244aeea907e086297f: Running coprocessor pre-open hook at 1731296481175Writing region info on filesystem at 1731296481175Initializing all the Stores at 1731296481177 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296481177Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296481177Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296481177Cleaning up temporary data from old regions at 1731296481190 (+13 ms)Running coprocessor post-open hooks at 1731296481197 (+7 ms)Region opened successfully at 1731296481198 (+1 ms) 2024-11-11T03:41:21,199 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., pid=93, masterSystemTime=1731296481165 2024-11-11T03:41:21,202 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:21,202 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:21,202 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=OPEN, openSeqNum=2, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:21,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:41:21,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T03:41:21,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 in 194 msec 2024-11-11T03:41:21,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T03:41:21,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, ASSIGN in 353 msec 2024-11-11T03:41:21,208 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T03:41:21,208 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296481208"}]},"ts":"1731296481208"} 2024-11-11T03:41:21,210 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T03:41:21,211 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T03:41:21,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 790 msec 2024-11-11T03:41:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T03:41:21,563 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:21,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:21,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:21,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:21,569 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df3a5d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:21,572 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7904f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:21,573 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:21,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:21,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T03:41:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ffe7f9d089e,45295,1731296383499 2024-11-11T03:41:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51612c3f 2024-11-11T03:41:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:41:21,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:41:21,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:41:21,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:41:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:21,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742170_1346 (size=1097) 2024-11-11T03:41:21,992 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T03:41:21,992 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T03:41:21,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:41:22,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, REOPEN/MOVE}] 2024-11-11T03:41:22,001 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, REOPEN/MOVE 2024-11-11T03:41:22,002 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, REOPEN/MOVE because future has completed 2024-11-11T03:41:22,004 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:41:22,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:41:22,157 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,158 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:41:22,158 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 4216aa280844bc244aeea907e086297f, disabling compactions & flushes 2024-11-11T03:41:22,158 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,158 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,158 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. after waiting 0 ms 2024-11-11T03:41:22,158 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,166 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T03:41:22,167 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,168 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 4216aa280844bc244aeea907e086297f: Waiting for close lock at 1731296482158Running coprocessor pre-close hooks at 1731296482158Disabling compacts and flushes for region at 1731296482158Disabling writes for close at 1731296482158Writing region close event to WAL at 1731296482159 (+1 ms)Running coprocessor post-close hooks at 1731296482167 (+8 ms)Closed at 1731296482167 2024-11-11T03:41:22,168 WARN [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 4216aa280844bc244aeea907e086297f to self. 2024-11-11T03:41:22,171 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,171 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=CLOSED 2024-11-11T03:41:22,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:41:22,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-11T03:41:22,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 in 170 msec 2024-11-11T03:41:22,178 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, REOPEN/MOVE; state=CLOSED, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=true 2024-11-11T03:41:22,328 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, REOPEN/MOVE because future has completed 2024-11-11T03:41:22,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:41:22,492 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,492 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:41:22,493 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,493 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:41:22,493 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,494 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,495 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,497 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:22,497 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4216aa280844bc244aeea907e086297f columnFamilyName A 2024-11-11T03:41:22,498 DEBUG [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:22,499 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(327): Store=4216aa280844bc244aeea907e086297f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:22,499 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,500 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:22,500 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4216aa280844bc244aeea907e086297f columnFamilyName B 2024-11-11T03:41:22,500 DEBUG [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:22,501 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(327): Store=4216aa280844bc244aeea907e086297f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:22,501 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,501 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:22,502 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4216aa280844bc244aeea907e086297f columnFamilyName C 2024-11-11T03:41:22,502 DEBUG [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:22,502 INFO [StoreOpener-4216aa280844bc244aeea907e086297f-1 {}] regionserver.HStore(327): Store=4216aa280844bc244aeea907e086297f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:22,502 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,502 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,503 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,505 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,507 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,507 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,507 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:41:22,509 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,510 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 4216aa280844bc244aeea907e086297f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67349525, jitterRate=0.003586128354072571}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:41:22,510 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,511 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 4216aa280844bc244aeea907e086297f: Running coprocessor pre-open hook at 1731296482494Writing region info on filesystem at 1731296482494Initializing all the Stores at 1731296482495 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296482495Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296482495Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296482495Cleaning up temporary data from old regions at 1731296482507 (+12 ms)Running coprocessor post-open hooks at 1731296482510 (+3 ms)Region opened successfully at 1731296482511 (+1 ms) 2024-11-11T03:41:22,512 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., pid=98, masterSystemTime=1731296482485 2024-11-11T03:41:22,514 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,515 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,515 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=OPEN, openSeqNum=5, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:41:22,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-11T03:41:22,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 in 187 msec 2024-11-11T03:41:22,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T03:41:22,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, REOPEN/MOVE in 520 msec 2024-11-11T03:41:22,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-11T03:41:22,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-11-11T03:41:22,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-11T03:41:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-11T03:41:22,530 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:22,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c3c76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,534 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332f12b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,537 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1554103e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d18ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,542 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e00ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458b4fab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,544 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd50596, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,545 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20234473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,547 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d064f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a53572c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,549 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab194cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,550 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6ea4f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,552 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e8804c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,553 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458fb910, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,554 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b609f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,556 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3ecc73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e878dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,560 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8b5e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:22,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:22,563 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@748349d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:22,564 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375a785f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:22,565 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,565 DEBUG [Thread-1587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,565 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,565 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:22,565 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,566 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,566 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T03:41:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,567 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T03:41:22,568 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,568 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,569 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,569 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:22,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,576 DEBUG [Thread-1589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:22,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:41:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:22,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:41:22,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ec5c5e7ba8074ca89d88565b980a7f05_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296482580/Put/seqid=0 2024-11-11T03:41:22,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742171_1347 (size=12154) 2024-11-11T03:41:22,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:22,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296542614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731296482601, max timestamp=1731296482620Num uniques 859; 2024-11-11T03:41:22,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731296482601, max timestamp=1731296482620Num uniques 859; 2024-11-11T03:41:22,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731296482601, max timestamp=1731296482620Num uniques 859; 2024-11-11T03:41:22,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T03:41:22,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T03:41:22,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T03:41:22,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ec5c5e7ba8074ca89d88565b980a7f05_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ec5c5e7ba8074ca89d88565b980a7f05_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:22,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/a66785fd89874bc88ce16688492c7581, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:22,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/a66785fd89874bc88ce16688492c7581 is 175, key is test_row_0/A:col10/1731296482580/Put/seqid=0 2024-11-11T03:41:22,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742172_1348 (size=30955) 2024-11-11T03:41:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296542646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296542647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296542655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296542655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T03:41:22,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:22,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:22,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:22,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:22,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296542732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296542761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296542761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296542762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296542762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:22,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:22,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:22,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:22,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T03:41:22,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296542942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296542971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296542972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296542972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296542972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/a66785fd89874bc88ce16688492c7581 2024-11-11T03:41:23,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/43c7cdb822cb44a8b7b7e0536ab60c76 is 50, key is test_row_0/B:col10/1731296482580/Put/seqid=0 2024-11-11T03:41:23,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742173_1349 (size=12001) 2024-11-11T03:41:23,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T03:41:23,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296543252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296543281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296543281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296543292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296543292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:23,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:23,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/43c7cdb822cb44a8b7b7e0536ab60c76 2024-11-11T03:41:23,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/d3f44a7745024418be3e4daaa68217c4 is 50, key is test_row_0/C:col10/1731296482580/Put/seqid=0 2024-11-11T03:41:23,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742174_1350 (size=12001) 2024-11-11T03:41:23,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T03:41:23,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296543771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296543792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296543792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296543812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296543812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:23,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/d3f44a7745024418be3e4daaa68217c4 2024-11-11T03:41:23,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/a66785fd89874bc88ce16688492c7581 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/a66785fd89874bc88ce16688492c7581 2024-11-11T03:41:23,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/a66785fd89874bc88ce16688492c7581, entries=150, sequenceid=15, filesize=30.2 K 2024-11-11T03:41:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/43c7cdb822cb44a8b7b7e0536ab60c76 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/43c7cdb822cb44a8b7b7e0536ab60c76 2024-11-11T03:41:23,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/43c7cdb822cb44a8b7b7e0536ab60c76, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T03:41:23,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/d3f44a7745024418be3e4daaa68217c4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d3f44a7745024418be3e4daaa68217c4 2024-11-11T03:41:23,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d3f44a7745024418be3e4daaa68217c4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T03:41:23,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4216aa280844bc244aeea907e086297f in 1315ms, sequenceid=15, compaction requested=false 2024-11-11T03:41:23,896 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T03:41:23,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:23,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T03:41:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:23,944 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-11T03:41:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ee3a64cb6214f17aaee40cc31865e80_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296482654/Put/seqid=0 2024-11-11T03:41:23,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742175_1351 (size=12154) 2024-11-11T03:41:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,357 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ee3a64cb6214f17aaee40cc31865e80_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ee3a64cb6214f17aaee40cc31865e80_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/12baad22401d41faac6ef8d08f6ae46f, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/12baad22401d41faac6ef8d08f6ae46f is 175, key is test_row_0/A:col10/1731296482654/Put/seqid=0 2024-11-11T03:41:24,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742176_1352 (size=30955) 2024-11-11T03:41:24,362 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/12baad22401d41faac6ef8d08f6ae46f 2024-11-11T03:41:24,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/ce4313f96edf41fea72c91456c34447c is 50, key is test_row_0/B:col10/1731296482654/Put/seqid=0 2024-11-11T03:41:24,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742177_1353 (size=12001) 2024-11-11T03:41:24,374 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/ce4313f96edf41fea72c91456c34447c 2024-11-11T03:41:24,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/3228aa00299643a092810cfcd0eddc27 is 50, key is test_row_0/C:col10/1731296482654/Put/seqid=0 2024-11-11T03:41:24,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742178_1354 (size=12001) 2024-11-11T03:41:24,388 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/3228aa00299643a092810cfcd0eddc27 2024-11-11T03:41:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/12baad22401d41faac6ef8d08f6ae46f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/12baad22401d41faac6ef8d08f6ae46f 2024-11-11T03:41:24,394 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/12baad22401d41faac6ef8d08f6ae46f, entries=150, sequenceid=48, filesize=30.2 K 2024-11-11T03:41:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/ce4313f96edf41fea72c91456c34447c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ce4313f96edf41fea72c91456c34447c 2024-11-11T03:41:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,398 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ce4313f96edf41fea72c91456c34447c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T03:41:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/3228aa00299643a092810cfcd0eddc27 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/3228aa00299643a092810cfcd0eddc27 2024-11-11T03:41:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/3228aa00299643a092810cfcd0eddc27, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,403 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=0 B/0 for 4216aa280844bc244aeea907e086297f in 458ms, sequenceid=48, compaction requested=false 2024-11-11T03:41:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T03:41:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T03:41:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T03:41:24,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8350 sec 2024-11-11T03:41:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8420 sec 2024-11-11T03:41:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,722 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,723 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T03:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,724 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T03:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,725 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T03:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T03:41:24,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,878 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T03:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fec67f4af81040ba8e5dcbd882f9114b_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296484867/Put/seqid=0 2024-11-11T03:41:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742179_1355 (size=14594) 2024-11-11T03:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,913 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fec67f4af81040ba8e5dcbd882f9114b_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fec67f4af81040ba8e5dcbd882f9114b_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/408f0acc985441318b9bf7265b09dc09, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/408f0acc985441318b9bf7265b09dc09 is 175, key is test_row_0/A:col10/1731296484867/Put/seqid=0 2024-11-11T03:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742180_1356 (size=39545) 2024-11-11T03:41:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/408f0acc985441318b9bf7265b09dc09 2024-11-11T03:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/5d7ae6e21e694f2d948f0e08c028da73 is 50, key is test_row_0/B:col10/1731296484867/Put/seqid=0 2024-11-11T03:41:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742181_1357 (size=9657) 2024-11-11T03:41:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,952 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/5d7ae6e21e694f2d948f0e08c028da73 2024-11-11T03:41:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e7b18eb547a64d9aae79f517b9db776c is 50, key is test_row_0/C:col10/1731296484867/Put/seqid=0 2024-11-11T03:41:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:24,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296484945, max timestamp=1731296484964Num uniques 859; 2024-11-11T03:41:24,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:41:24,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296484945, max timestamp=1731296484964Num uniques 859; 2024-11-11T03:41:24,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296484945, max timestamp=1731296484964Num uniques 859; 2024-11-11T03:41:24,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:41:24,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:41:24,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742182_1358 (size=9657) 2024-11-11T03:41:24,975 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e7b18eb547a64d9aae79f517b9db776c 2024-11-11T03:41:24,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/408f0acc985441318b9bf7265b09dc09 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/408f0acc985441318b9bf7265b09dc09 2024-11-11T03:41:24,983 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/408f0acc985441318b9bf7265b09dc09, entries=200, sequenceid=55, filesize=38.6 K 2024-11-11T03:41:24,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/5d7ae6e21e694f2d948f0e08c028da73 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/5d7ae6e21e694f2d948f0e08c028da73 2024-11-11T03:41:24,991 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/5d7ae6e21e694f2d948f0e08c028da73, entries=100, sequenceid=55, filesize=9.4 K 2024-11-11T03:41:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e7b18eb547a64d9aae79f517b9db776c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e7b18eb547a64d9aae79f517b9db776c 2024-11-11T03:41:24,995 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e7b18eb547a64d9aae79f517b9db776c, entries=100, sequenceid=55, filesize=9.4 K 2024-11-11T03:41:24,998 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=80.51 KB/82440 for 4216aa280844bc244aeea907e086297f in 121ms, sequenceid=55, compaction requested=true 2024-11-11T03:41:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T03:41:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:24,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=87.22 KB heapSize=240.84 KB 2024-11-11T03:41:24,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:24,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:24,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:24,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:24,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:24,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T03:41:25,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T03:41:25,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 275 msec 2024-11-11T03:41:25,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 282 msec 2024-11-11T03:41:25,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b5dfe3b8fe4e4ac5abccef39e9b880b4_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296484988/Put/seqid=0 2024-11-11T03:41:25,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742183_1359 (size=19474) 2024-11-11T03:41:25,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:25,039 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b5dfe3b8fe4e4ac5abccef39e9b880b4_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b5dfe3b8fe4e4ac5abccef39e9b880b4_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:25,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/37542e8faf4e4b97a99c40fff5307639, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:25,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/37542e8faf4e4b97a99c40fff5307639 is 175, key is test_row_0/A:col10/1731296484988/Put/seqid=0 2024-11-11T03:41:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T03:41:25,043 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:25,043 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-11T03:41:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T03:41:25,045 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:25,046 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:25,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:25,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742184_1360 (size=56733) 2024-11-11T03:41:25,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/37542e8faf4e4b97a99c40fff5307639 2024-11-11T03:41:25,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296545039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296545044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296545057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2a43071baa994a5aa72ccb17535c04ab is 50, key is test_row_0/B:col10/1731296484988/Put/seqid=0 2024-11-11T03:41:25,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742185_1361 (size=12001) 2024-11-11T03:41:25,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296545062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296545064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T03:41:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296545171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296545171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296545171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296545182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T03:41:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296545192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,290 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:41:25,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T03:41:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T03:41:25,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296545392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296545392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296545392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296545402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296545402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2a43071baa994a5aa72ccb17535c04ab 2024-11-11T03:41:25,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/65ce31dc1bcb4d5abc6321ba7fa5894a is 50, key is test_row_0/C:col10/1731296484988/Put/seqid=0 2024-11-11T03:41:25,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742186_1362 (size=12001) 2024-11-11T03:41:25,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T03:41:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T03:41:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T03:41:25,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296545712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296545712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296545712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296545712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:25,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296545712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:25,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T03:41:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:25,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/65ce31dc1bcb4d5abc6321ba7fa5894a 2024-11-11T03:41:25,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/37542e8faf4e4b97a99c40fff5307639 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/37542e8faf4e4b97a99c40fff5307639 2024-11-11T03:41:25,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/37542e8faf4e4b97a99c40fff5307639, entries=300, sequenceid=84, filesize=55.4 K 2024-11-11T03:41:25,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2a43071baa994a5aa72ccb17535c04ab as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2a43071baa994a5aa72ccb17535c04ab 2024-11-11T03:41:25,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2a43071baa994a5aa72ccb17535c04ab, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T03:41:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/65ce31dc1bcb4d5abc6321ba7fa5894a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/65ce31dc1bcb4d5abc6321ba7fa5894a 2024-11-11T03:41:25,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/65ce31dc1bcb4d5abc6321ba7fa5894a, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T03:41:25,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~240.80 KB/246576, currentSize=100.63 KB/103050 for 4216aa280844bc244aeea907e086297f in 899ms, sequenceid=84, compaction requested=true 2024-11-11T03:41:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:25,897 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:41:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:25,897 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:41:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:25,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:25,898 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:41:25,898 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:41:25,898 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/A is initiating minor compaction (all files) 2024-11-11T03:41:25,898 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/B is initiating minor compaction (all files) 2024-11-11T03:41:25,898 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/A in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,898 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/B in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,898 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/a66785fd89874bc88ce16688492c7581, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/12baad22401d41faac6ef8d08f6ae46f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/408f0acc985441318b9bf7265b09dc09, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/37542e8faf4e4b97a99c40fff5307639] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=154.5 K 2024-11-11T03:41:25,898 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/43c7cdb822cb44a8b7b7e0536ab60c76, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ce4313f96edf41fea72c91456c34447c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/5d7ae6e21e694f2d948f0e08c028da73, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2a43071baa994a5aa72ccb17535c04ab] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=44.6 K 2024-11-11T03:41:25,898 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,898 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/a66785fd89874bc88ce16688492c7581, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/12baad22401d41faac6ef8d08f6ae46f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/408f0acc985441318b9bf7265b09dc09, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/37542e8faf4e4b97a99c40fff5307639] 2024-11-11T03:41:25,898 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66785fd89874bc88ce16688492c7581, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731296482579 2024-11-11T03:41:25,898 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c7cdb822cb44a8b7b7e0536ab60c76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731296482579 2024-11-11T03:41:25,899 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12baad22401d41faac6ef8d08f6ae46f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731296482642 2024-11-11T03:41:25,899 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ce4313f96edf41fea72c91456c34447c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731296482642 2024-11-11T03:41:25,899 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d7ae6e21e694f2d948f0e08c028da73, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731296484858 2024-11-11T03:41:25,899 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408f0acc985441318b9bf7265b09dc09, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731296484823 2024-11-11T03:41:25,899 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37542e8faf4e4b97a99c40fff5307639, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731296484969 2024-11-11T03:41:25,899 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a43071baa994a5aa72ccb17535c04ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731296484986 2024-11-11T03:41:25,907 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:25,909 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111cabbc317fbc1427483464c7911d9188d_4216aa280844bc244aeea907e086297f store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:25,909 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:25,910 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/40382abc2afa4960be3777e65bbf229c is 50, key is test_row_0/B:col10/1731296484988/Put/seqid=0 2024-11-11T03:41:25,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111cabbc317fbc1427483464c7911d9188d_4216aa280844bc244aeea907e086297f, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:25,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cabbc317fbc1427483464c7911d9188d_4216aa280844bc244aeea907e086297f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:25,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742187_1363 (size=12139) 2024-11-11T03:41:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742188_1364 (size=4469) 2024-11-11T03:41:25,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T03:41:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:25,962 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T03:41:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:25,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110736190975794244ba2c8bca3fa20cd8_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296485038/Put/seqid=0 2024-11-11T03:41:25,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742189_1365 (size=12154) 2024-11-11T03:41:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T03:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296546248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296546255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296546255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296546256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296546257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,321 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/40382abc2afa4960be3777e65bbf229c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/40382abc2afa4960be3777e65bbf229c 2024-11-11T03:41:26,325 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4216aa280844bc244aeea907e086297f/B of 4216aa280844bc244aeea907e086297f into 40382abc2afa4960be3777e65bbf229c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:26,325 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:26,325 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/B, priority=12, startTime=1731296485897; duration=0sec 2024-11-11T03:41:26,325 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:26,325 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:B 2024-11-11T03:41:26,325 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:41:26,326 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:41:26,326 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/C is initiating minor compaction (all files) 2024-11-11T03:41:26,326 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/C in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:26,326 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d3f44a7745024418be3e4daaa68217c4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/3228aa00299643a092810cfcd0eddc27, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e7b18eb547a64d9aae79f517b9db776c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/65ce31dc1bcb4d5abc6321ba7fa5894a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=44.6 K 2024-11-11T03:41:26,326 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d3f44a7745024418be3e4daaa68217c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731296482579 2024-11-11T03:41:26,327 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3228aa00299643a092810cfcd0eddc27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731296482642 2024-11-11T03:41:26,327 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b18eb547a64d9aae79f517b9db776c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731296484858 2024-11-11T03:41:26,327 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ce31dc1bcb4d5abc6321ba7fa5894a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731296484986 2024-11-11T03:41:26,330 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:26,330 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/265441ce8b064957be54b97d81f9ddb4 is 175, key is test_row_0/A:col10/1731296484988/Put/seqid=0 2024-11-11T03:41:26,334 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:26,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742190_1366 (size=31093) 2024-11-11T03:41:26,334 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/03da2b87b3b34a10a7bd67dc895f6b36 is 50, key is test_row_0/C:col10/1731296484988/Put/seqid=0 2024-11-11T03:41:26,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742191_1367 (size=12139) 2024-11-11T03:41:26,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:26,375 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110736190975794244ba2c8bca3fa20cd8_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110736190975794244ba2c8bca3fa20cd8_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/2a628c02ea594c838807f2715390da06, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/2a628c02ea594c838807f2715390da06 is 175, key is test_row_0/A:col10/1731296485038/Put/seqid=0 2024-11-11T03:41:26,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742192_1368 (size=30955) 2024-11-11T03:41:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296546371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296546371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296546371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296546382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296546382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296546592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296546592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296546592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296546602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296546602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,739 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/265441ce8b064957be54b97d81f9ddb4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/265441ce8b064957be54b97d81f9ddb4 2024-11-11T03:41:26,741 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/03da2b87b3b34a10a7bd67dc895f6b36 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/03da2b87b3b34a10a7bd67dc895f6b36 2024-11-11T03:41:26,743 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4216aa280844bc244aeea907e086297f/A of 4216aa280844bc244aeea907e086297f into 265441ce8b064957be54b97d81f9ddb4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:26,743 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:26,743 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/A, priority=12, startTime=1731296485897; duration=0sec 2024-11-11T03:41:26,743 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:26,743 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:A 2024-11-11T03:41:26,744 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4216aa280844bc244aeea907e086297f/C of 4216aa280844bc244aeea907e086297f into 03da2b87b3b34a10a7bd67dc895f6b36(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:26,744 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:26,744 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/C, priority=12, startTime=1731296485897; duration=0sec 2024-11-11T03:41:26,745 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:26,745 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:C 2024-11-11T03:41:26,779 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/2a628c02ea594c838807f2715390da06 2024-11-11T03:41:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/03c8f6d328084076a93a8546a49c5f32 is 50, key is test_row_0/B:col10/1731296485038/Put/seqid=0 2024-11-11T03:41:26,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742193_1369 (size=12001) 2024-11-11T03:41:26,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296546912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296546912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296546912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296546912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296546912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:27,188 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/03c8f6d328084076a93a8546a49c5f32 2024-11-11T03:41:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/503cd77b2dc24eb388e815dbdbb5a173 is 50, key is test_row_0/C:col10/1731296485038/Put/seqid=0 2024-11-11T03:41:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742194_1370 (size=12001) 2024-11-11T03:41:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T03:41:27,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296547422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:27,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:27,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296547422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296547431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296547431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296547431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:27,599 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/503cd77b2dc24eb388e815dbdbb5a173 2024-11-11T03:41:27,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/2a628c02ea594c838807f2715390da06 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/2a628c02ea594c838807f2715390da06 2024-11-11T03:41:27,605 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/2a628c02ea594c838807f2715390da06, entries=150, sequenceid=102, filesize=30.2 K 2024-11-11T03:41:27,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/03c8f6d328084076a93a8546a49c5f32 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/03c8f6d328084076a93a8546a49c5f32 2024-11-11T03:41:27,608 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/03c8f6d328084076a93a8546a49c5f32, entries=150, sequenceid=102, filesize=11.7 K 2024-11-11T03:41:27,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/503cd77b2dc24eb388e815dbdbb5a173 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/503cd77b2dc24eb388e815dbdbb5a173 2024-11-11T03:41:27,612 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/503cd77b2dc24eb388e815dbdbb5a173, entries=150, sequenceid=102, filesize=11.7 K 2024-11-11T03:41:27,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4216aa280844bc244aeea907e086297f in 1651ms, sequenceid=102, compaction requested=false 2024-11-11T03:41:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-11T03:41:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-11T03:41:27,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T03:41:27,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5680 sec 2024-11-11T03:41:27,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.5740 sec 2024-11-11T03:41:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:28,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:41:28,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:28,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:28,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:28,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:28,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:28,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:28,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296548474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296548474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296548475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296548476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296548479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c77d986eebc34c1cac8a61693355ce0c_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296486256/Put/seqid=0 2024-11-11T03:41:28,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742195_1371 (size=14594) 2024-11-11T03:41:28,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296548591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296548592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296548592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296548592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296548592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296548801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296548812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296548812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296548812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296548812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:28,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:28,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c77d986eebc34c1cac8a61693355ce0c_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c77d986eebc34c1cac8a61693355ce0c_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:28,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/6ecf1c6c26584bcd8d74ab1d9d079e5f, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:28,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/6ecf1c6c26584bcd8d74ab1d9d079e5f is 175, key is test_row_0/A:col10/1731296486256/Put/seqid=0 2024-11-11T03:41:28,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742196_1372 (size=39549) 2024-11-11T03:41:29,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296549121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296549131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296549132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296549132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296549132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T03:41:29,222 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:29,222 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T03:41:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T03:41:29,224 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:29,224 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:29,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:29,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/6ecf1c6c26584bcd8d74ab1d9d079e5f 2024-11-11T03:41:29,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a15963514d8741798a2fc6c532091b8e is 50, key is test_row_0/B:col10/1731296486256/Put/seqid=0 2024-11-11T03:41:29,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742197_1373 (size=12001) 2024-11-11T03:41:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T03:41:29,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T03:41:29,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:29,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T03:41:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T03:41:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296549642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296549652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296549652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296549652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296549652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:29,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:29,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T03:41:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a15963514d8741798a2fc6c532091b8e 2024-11-11T03:41:29,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/4f6a32f8379a42d88049b013c5e9f5c4 is 50, key is test_row_0/C:col10/1731296486256/Put/seqid=0 2024-11-11T03:41:29,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742198_1374 (size=12001) 2024-11-11T03:41:29,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T03:41:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:29,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T03:41:29,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T03:41:29,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:29,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:29,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:30,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/4f6a32f8379a42d88049b013c5e9f5c4 2024-11-11T03:41:30,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/6ecf1c6c26584bcd8d74ab1d9d079e5f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/6ecf1c6c26584bcd8d74ab1d9d079e5f 2024-11-11T03:41:30,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/6ecf1c6c26584bcd8d74ab1d9d079e5f, entries=200, sequenceid=120, filesize=38.6 K 2024-11-11T03:41:30,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a15963514d8741798a2fc6c532091b8e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a15963514d8741798a2fc6c532091b8e 2024-11-11T03:41:30,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a15963514d8741798a2fc6c532091b8e, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T03:41:30,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/4f6a32f8379a42d88049b013c5e9f5c4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/4f6a32f8379a42d88049b013c5e9f5c4 2024-11-11T03:41:30,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T03:41:30,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:30,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:30,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:30,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:30,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:30,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/4f6a32f8379a42d88049b013c5e9f5c4, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T03:41:30,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4216aa280844bc244aeea907e086297f in 1700ms, sequenceid=120, compaction requested=true 2024-11-11T03:41:30,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:30,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:30,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:30,146 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:30,146 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:30,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:30,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:30,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:30,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:30,147 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:30,147 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/B is initiating minor compaction (all files) 2024-11-11T03:41:30,147 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/B in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:30,147 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/40382abc2afa4960be3777e65bbf229c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/03c8f6d328084076a93a8546a49c5f32, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a15963514d8741798a2fc6c532091b8e] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=35.3 K 2024-11-11T03:41:30,147 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:30,148 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/A is initiating minor compaction (all files) 2024-11-11T03:41:30,148 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/A in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:30,148 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/265441ce8b064957be54b97d81f9ddb4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/2a628c02ea594c838807f2715390da06, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/6ecf1c6c26584bcd8d74ab1d9d079e5f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=99.2 K 2024-11-11T03:41:30,148 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:30,148 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/265441ce8b064957be54b97d81f9ddb4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/2a628c02ea594c838807f2715390da06, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/6ecf1c6c26584bcd8d74ab1d9d079e5f] 2024-11-11T03:41:30,148 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 40382abc2afa4960be3777e65bbf229c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731296484986 2024-11-11T03:41:30,148 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 265441ce8b064957be54b97d81f9ddb4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731296484986 2024-11-11T03:41:30,148 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a628c02ea594c838807f2715390da06, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731296485038 2024-11-11T03:41:30,148 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c8f6d328084076a93a8546a49c5f32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731296485038 2024-11-11T03:41:30,149 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting a15963514d8741798a2fc6c532091b8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731296486255 2024-11-11T03:41:30,149 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ecf1c6c26584bcd8d74ab1d9d079e5f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731296486253 2024-11-11T03:41:30,157 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:30,161 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:30,161 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/b9ea154364a64913bb9169449c8d1487 is 50, key is test_row_0/B:col10/1731296486256/Put/seqid=0 2024-11-11T03:41:30,161 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110201087b58a3461e8e080095c65cef8f_4216aa280844bc244aeea907e086297f store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:30,163 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110201087b58a3461e8e080095c65cef8f_4216aa280844bc244aeea907e086297f, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:30,163 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110201087b58a3461e8e080095c65cef8f_4216aa280844bc244aeea907e086297f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:30,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742199_1375 (size=12241) 2024-11-11T03:41:30,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742200_1376 (size=4469) 2024-11-11T03:41:30,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T03:41:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:30,293 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:41:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:30,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111143489b6698c64be8959e4ac1b7e1e26e_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296488477/Put/seqid=0 2024-11-11T03:41:30,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742201_1377 (size=12304) 2024-11-11T03:41:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T03:41:30,575 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/b9ea154364a64913bb9169449c8d1487 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/b9ea154364a64913bb9169449c8d1487 2024-11-11T03:41:30,578 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#A#compaction#317 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:30,578 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/de047e877413475b87e4bdca90b2b04b is 175, key is test_row_0/A:col10/1731296486256/Put/seqid=0 2024-11-11T03:41:30,579 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/B of 4216aa280844bc244aeea907e086297f into b9ea154364a64913bb9169449c8d1487(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:30,580 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:30,580 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/B, priority=13, startTime=1731296490145; duration=0sec 2024-11-11T03:41:30,580 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:30,580 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:B 2024-11-11T03:41:30,580 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:30,581 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:30,581 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/C is initiating minor compaction (all files) 2024-11-11T03:41:30,581 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/C in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:30,581 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/03da2b87b3b34a10a7bd67dc895f6b36, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/503cd77b2dc24eb388e815dbdbb5a173, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/4f6a32f8379a42d88049b013c5e9f5c4] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=35.3 K 2024-11-11T03:41:30,581 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 03da2b87b3b34a10a7bd67dc895f6b36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731296484986 2024-11-11T03:41:30,582 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 503cd77b2dc24eb388e815dbdbb5a173, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731296485038 2024-11-11T03:41:30,582 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f6a32f8379a42d88049b013c5e9f5c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731296486255 2024-11-11T03:41:30,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742202_1378 (size=31195) 2024-11-11T03:41:30,588 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:30,589 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/6afd3396342c423fae043ebdffa36ef1 is 50, key is test_row_0/C:col10/1731296486256/Put/seqid=0 2024-11-11T03:41:30,591 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/de047e877413475b87e4bdca90b2b04b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/de047e877413475b87e4bdca90b2b04b 2024-11-11T03:41:30,595 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/A of 4216aa280844bc244aeea907e086297f into de047e877413475b87e4bdca90b2b04b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:30,595 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:30,595 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/A, priority=13, startTime=1731296490145; duration=0sec 2024-11-11T03:41:30,595 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:30,595 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:A 2024-11-11T03:41:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742203_1379 (size=12241) 2024-11-11T03:41:30,599 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/6afd3396342c423fae043ebdffa36ef1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/6afd3396342c423fae043ebdffa36ef1 2024-11-11T03:41:30,605 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/C of 4216aa280844bc244aeea907e086297f into 6afd3396342c423fae043ebdffa36ef1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:30,605 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:30,605 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/C, priority=13, startTime=1731296490146; duration=0sec 2024-11-11T03:41:30,605 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:30,606 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:C 2024-11-11T03:41:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:30,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:30,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296550690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:30,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296550691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296550692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296550693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296550693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111143489b6698c64be8959e4ac1b7e1e26e_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111143489b6698c64be8959e4ac1b7e1e26e_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:30,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/7c073d3a8bdd4b1b968ccd7af2bb7d44, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:30,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/7c073d3a8bdd4b1b968ccd7af2bb7d44 is 175, key is test_row_0/A:col10/1731296488477/Put/seqid=0 2024-11-11T03:41:30,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742204_1380 (size=31105) 2024-11-11T03:41:30,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296550801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296550812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296550812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296550812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296550812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296551022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296551032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,110 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/7c073d3a8bdd4b1b968ccd7af2bb7d44 2024-11-11T03:41:31,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/ba5a64af373e4a6282113f59a7dac79a is 50, key is test_row_0/B:col10/1731296488477/Put/seqid=0 2024-11-11T03:41:31,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742205_1381 (size=12151) 2024-11-11T03:41:31,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296551341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296551341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296551342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296551342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296551342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T03:41:31,522 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/ba5a64af373e4a6282113f59a7dac79a 2024-11-11T03:41:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e6b6ffa172c74f3cb7135741bfecaea5 is 50, key is test_row_0/C:col10/1731296488477/Put/seqid=0 2024-11-11T03:41:31,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742206_1382 (size=12151) 2024-11-11T03:41:31,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296551851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296551851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296551851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296551852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296551862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:31,931 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e6b6ffa172c74f3cb7135741bfecaea5 2024-11-11T03:41:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/7c073d3a8bdd4b1b968ccd7af2bb7d44 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/7c073d3a8bdd4b1b968ccd7af2bb7d44 2024-11-11T03:41:31,941 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/7c073d3a8bdd4b1b968ccd7af2bb7d44, entries=150, sequenceid=137, filesize=30.4 K 2024-11-11T03:41:31,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/ba5a64af373e4a6282113f59a7dac79a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ba5a64af373e4a6282113f59a7dac79a 2024-11-11T03:41:31,946 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ba5a64af373e4a6282113f59a7dac79a, entries=150, sequenceid=137, filesize=11.9 K 2024-11-11T03:41:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e6b6ffa172c74f3cb7135741bfecaea5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e6b6ffa172c74f3cb7135741bfecaea5 2024-11-11T03:41:31,952 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e6b6ffa172c74f3cb7135741bfecaea5, entries=150, sequenceid=137, filesize=11.9 K 2024-11-11T03:41:31,953 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4216aa280844bc244aeea907e086297f in 1660ms, sequenceid=137, compaction requested=false 2024-11-11T03:41:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T03:41:31,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T03:41:31,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T03:41:31,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7310 sec 2024-11-11T03:41:31,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.7350 sec 2024-11-11T03:41:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:32,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:41:32,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:32,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:32,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:32,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:32,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a471d5546faf4b539f93e0f07cbef086_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296492864/Put/seqid=0 2024-11-11T03:41:32,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742207_1383 (size=14794) 2024-11-11T03:41:32,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296552891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:32,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296552892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:32,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296552892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296552893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296552894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296553002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296553012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296553012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296553012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296553012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296553212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296553222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296553222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296553222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296553222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,275 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:33,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a471d5546faf4b539f93e0f07cbef086_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a471d5546faf4b539f93e0f07cbef086_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:33,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/23a906e7715b436ea56dd6e3262c0276, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:33,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/23a906e7715b436ea56dd6e3262c0276 is 175, key is test_row_0/A:col10/1731296492864/Put/seqid=0 2024-11-11T03:41:33,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742208_1384 (size=39749) 2024-11-11T03:41:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T03:41:33,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:33,393 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T03:41:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T03:41:33,394 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:33,395 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:33,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T03:41:33,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296553522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296553531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296553531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296553531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296553532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:33,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:33,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/23a906e7715b436ea56dd6e3262c0276 2024-11-11T03:41:33,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/cbd4c05c46404c40baec4cf11413574d is 50, key is test_row_0/B:col10/1731296492864/Put/seqid=0 2024-11-11T03:41:33,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:33,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:33,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742209_1385 (size=12151) 2024-11-11T03:41:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T03:41:33,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:33,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:33,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:34,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:34,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T03:41:34,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296554042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:34,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296554042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:34,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296554052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:34,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296554052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:34,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296554052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:34,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/cbd4c05c46404c40baec4cf11413574d 2024-11-11T03:41:34,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/00a0d25f404345429d8d1c6dfc90de7b is 50, key is test_row_0/C:col10/1731296492864/Put/seqid=0 2024-11-11T03:41:34,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742210_1386 (size=12151) 2024-11-11T03:41:34,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:34,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:34,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/00a0d25f404345429d8d1c6dfc90de7b 2024-11-11T03:41:34,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/23a906e7715b436ea56dd6e3262c0276 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/23a906e7715b436ea56dd6e3262c0276 2024-11-11T03:41:34,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/23a906e7715b436ea56dd6e3262c0276, entries=200, sequenceid=157, filesize=38.8 K 2024-11-11T03:41:34,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/cbd4c05c46404c40baec4cf11413574d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/cbd4c05c46404c40baec4cf11413574d 2024-11-11T03:41:34,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/cbd4c05c46404c40baec4cf11413574d, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T03:41:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/00a0d25f404345429d8d1c6dfc90de7b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/00a0d25f404345429d8d1c6dfc90de7b 2024-11-11T03:41:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T03:41:34,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/00a0d25f404345429d8d1c6dfc90de7b, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T03:41:34,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4216aa280844bc244aeea907e086297f in 1669ms, sequenceid=157, compaction requested=true 2024-11-11T03:41:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:34,534 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:34,534 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:34,534 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/A is initiating minor compaction (all files) 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:34,535 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/A in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/B is initiating minor compaction (all files) 2024-11-11T03:41:34,535 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/de047e877413475b87e4bdca90b2b04b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/7c073d3a8bdd4b1b968ccd7af2bb7d44, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/23a906e7715b436ea56dd6e3262c0276] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=99.7 K 2024-11-11T03:41:34,535 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/B in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,535 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/de047e877413475b87e4bdca90b2b04b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/7c073d3a8bdd4b1b968ccd7af2bb7d44, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/23a906e7715b436ea56dd6e3262c0276] 2024-11-11T03:41:34,535 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/b9ea154364a64913bb9169449c8d1487, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ba5a64af373e4a6282113f59a7dac79a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/cbd4c05c46404c40baec4cf11413574d] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=35.7 K 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting de047e877413475b87e4bdca90b2b04b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731296486255 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ea154364a64913bb9169449c8d1487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731296486255 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c073d3a8bdd4b1b968ccd7af2bb7d44, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731296488465 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ba5a64af373e4a6282113f59a7dac79a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731296488465 2024-11-11T03:41:34,535 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a906e7715b436ea56dd6e3262c0276, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731296490691 2024-11-11T03:41:34,536 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd4c05c46404c40baec4cf11413574d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731296490691 2024-11-11T03:41:34,540 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:34,542 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:34,543 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/8994d9dcf4134a3d8ccba864218c76ee is 50, key is test_row_0/B:col10/1731296492864/Put/seqid=0 2024-11-11T03:41:34,543 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411119eebb2281b1d40fe84ddd042fe48986f_4216aa280844bc244aeea907e086297f store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:34,545 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411119eebb2281b1d40fe84ddd042fe48986f_4216aa280844bc244aeea907e086297f, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:34,545 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119eebb2281b1d40fe84ddd042fe48986f_4216aa280844bc244aeea907e086297f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:34,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742211_1387 (size=12493) 2024-11-11T03:41:34,560 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/8994d9dcf4134a3d8ccba864218c76ee as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/8994d9dcf4134a3d8ccba864218c76ee 2024-11-11T03:41:34,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742212_1388 (size=4469) 2024-11-11T03:41:34,564 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/B of 4216aa280844bc244aeea907e086297f into 8994d9dcf4134a3d8ccba864218c76ee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:34,564 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:34,564 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/B, priority=13, startTime=1731296494534; duration=0sec 2024-11-11T03:41:34,564 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:34,564 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:B 2024-11-11T03:41:34,564 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:34,565 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:34,565 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/C is initiating minor compaction (all files) 2024-11-11T03:41:34,565 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/C in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,565 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/6afd3396342c423fae043ebdffa36ef1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e6b6ffa172c74f3cb7135741bfecaea5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/00a0d25f404345429d8d1c6dfc90de7b] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=35.7 K 2024-11-11T03:41:34,565 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6afd3396342c423fae043ebdffa36ef1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731296486255 2024-11-11T03:41:34,566 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#A#compaction#326 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:34,566 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b6ffa172c74f3cb7135741bfecaea5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731296488465 2024-11-11T03:41:34,566 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a0d25f404345429d8d1c6dfc90de7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731296490691 2024-11-11T03:41:34,566 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/fed79a879fec482dadf51a507866dcc0 is 175, key is test_row_0/A:col10/1731296492864/Put/seqid=0 2024-11-11T03:41:34,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742213_1389 (size=31447) 2024-11-11T03:41:34,572 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:34,573 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/f86a62e2eccf4f4591adb9a9e98b7787 is 50, key is test_row_0/C:col10/1731296492864/Put/seqid=0 2024-11-11T03:41:34,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742214_1390 (size=12493) 2024-11-11T03:41:34,579 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/fed79a879fec482dadf51a507866dcc0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/fed79a879fec482dadf51a507866dcc0 2024-11-11T03:41:34,582 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/f86a62e2eccf4f4591adb9a9e98b7787 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/f86a62e2eccf4f4591adb9a9e98b7787 2024-11-11T03:41:34,583 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/A of 4216aa280844bc244aeea907e086297f into fed79a879fec482dadf51a507866dcc0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:34,583 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:34,583 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/A, priority=13, startTime=1731296494534; duration=0sec 2024-11-11T03:41:34,583 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:34,583 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:A 2024-11-11T03:41:34,585 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/C of 4216aa280844bc244aeea907e086297f into f86a62e2eccf4f4591adb9a9e98b7787(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:34,585 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:34,585 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/C, priority=13, startTime=1731296494534; duration=0sec 2024-11-11T03:41:34,585 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:34,585 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:C 2024-11-11T03:41:34,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T03:41:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:34,615 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:41:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:34,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ebcfa5a28efe4e4a8e5071034e6fce91_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296492893/Put/seqid=0 2024-11-11T03:41:34,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742215_1391 (size=12304) 2024-11-11T03:41:34,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:34,630 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ebcfa5a28efe4e4a8e5071034e6fce91_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebcfa5a28efe4e4a8e5071034e6fce91_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/69d3496271d14cd7b4c45f09ff4af215, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/69d3496271d14cd7b4c45f09ff4af215 is 175, key is test_row_0/A:col10/1731296492893/Put/seqid=0 2024-11-11T03:41:34,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742216_1392 (size=31105) 2024-11-11T03:41:34,634 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/69d3496271d14cd7b4c45f09ff4af215 2024-11-11T03:41:34,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2e1f62f8171c4149934eaa46f9a1e6ca is 50, key is test_row_0/B:col10/1731296492893/Put/seqid=0 2024-11-11T03:41:34,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742217_1393 (size=12151) 2024-11-11T03:41:35,044 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2e1f62f8171c4149934eaa46f9a1e6ca 2024-11-11T03:41:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/2ae7a5396ad74da8a237f7cce77916ee is 50, key is test_row_0/C:col10/1731296492893/Put/seqid=0 2024-11-11T03:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:35,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:35,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742218_1394 (size=12151) 2024-11-11T03:41:35,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296555080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296555081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296555081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296555082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296555086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296555191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296555192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296555192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296555202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296555202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296555412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296555412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296555412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296555412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296555422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,456 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/2ae7a5396ad74da8a237f7cce77916ee 2024-11-11T03:41:35,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/69d3496271d14cd7b4c45f09ff4af215 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/69d3496271d14cd7b4c45f09ff4af215 2024-11-11T03:41:35,462 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/69d3496271d14cd7b4c45f09ff4af215, entries=150, sequenceid=175, filesize=30.4 K 2024-11-11T03:41:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2e1f62f8171c4149934eaa46f9a1e6ca as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e1f62f8171c4149934eaa46f9a1e6ca 2024-11-11T03:41:35,465 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e1f62f8171c4149934eaa46f9a1e6ca, entries=150, sequenceid=175, filesize=11.9 K 2024-11-11T03:41:35,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/2ae7a5396ad74da8a237f7cce77916ee as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/2ae7a5396ad74da8a237f7cce77916ee 2024-11-11T03:41:35,469 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/2ae7a5396ad74da8a237f7cce77916ee, entries=150, sequenceid=175, filesize=11.9 K 2024-11-11T03:41:35,469 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4216aa280844bc244aeea907e086297f in 854ms, sequenceid=175, compaction requested=false 2024-11-11T03:41:35,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:35,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:35,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T03:41:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T03:41:35,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T03:41:35,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0760 sec 2024-11-11T03:41:35,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0790 sec 2024-11-11T03:41:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T03:41:35,542 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:35,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T03:41:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T03:41:35,544 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:35,545 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:35,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T03:41:35,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T03:41:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:35,697 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:41:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119534e6c7ae4e4734a14a695eed5d2243_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296495078/Put/seqid=0 2024-11-11T03:41:35,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742219_1395 (size=12304) 2024-11-11T03:41:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:35,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296555742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296555742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296555743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296555743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296555752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T03:41:35,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296555861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296555861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296555861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296555862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296556072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296556081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296556081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296556081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:36,111 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119534e6c7ae4e4734a14a695eed5d2243_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119534e6c7ae4e4734a14a695eed5d2243_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:36,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/97096e41f11f4098a61408429a36ba87, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:36,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/97096e41f11f4098a61408429a36ba87 is 175, key is test_row_0/A:col10/1731296495078/Put/seqid=0 2024-11-11T03:41:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742220_1396 (size=31105) 2024-11-11T03:41:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T03:41:36,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296556262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296556382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296556391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296556391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296556391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/97096e41f11f4098a61408429a36ba87 2024-11-11T03:41:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a1334fbb18af4f439c10db98fa30a957 is 50, key is test_row_0/B:col10/1731296495078/Put/seqid=0 2024-11-11T03:41:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742221_1397 (size=12151) 2024-11-11T03:41:36,525 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a1334fbb18af4f439c10db98fa30a957 2024-11-11T03:41:36,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/1a31c55f98f640f0871257713606520b is 50, key is test_row_0/C:col10/1731296495078/Put/seqid=0 2024-11-11T03:41:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742222_1398 (size=12151) 2024-11-11T03:41:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T03:41:36,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296556891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296556912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296556912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296556912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:36,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/1a31c55f98f640f0871257713606520b 2024-11-11T03:41:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/97096e41f11f4098a61408429a36ba87 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/97096e41f11f4098a61408429a36ba87 2024-11-11T03:41:36,941 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/97096e41f11f4098a61408429a36ba87, entries=150, sequenceid=192, filesize=30.4 K 2024-11-11T03:41:36,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a1334fbb18af4f439c10db98fa30a957 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a1334fbb18af4f439c10db98fa30a957 2024-11-11T03:41:36,944 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a1334fbb18af4f439c10db98fa30a957, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T03:41:36,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/1a31c55f98f640f0871257713606520b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/1a31c55f98f640f0871257713606520b 2024-11-11T03:41:36,947 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/1a31c55f98f640f0871257713606520b, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T03:41:36,948 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4216aa280844bc244aeea907e086297f in 1251ms, sequenceid=192, compaction requested=true 2024-11-11T03:41:36,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:36,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:36,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T03:41:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T03:41:36,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T03:41:36,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-11T03:41:36,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4080 sec 2024-11-11T03:41:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:37,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T03:41:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:37,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:37,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111114c298fbe9db47a0baea89d238a6913a_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296495751/Put/seqid=0 2024-11-11T03:41:37,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742223_1399 (size=14794) 2024-11-11T03:41:37,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296557385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:37,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296557501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T03:41:37,692 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:37,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T03:41:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T03:41:37,694 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:37,695 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:37,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:37,697 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:37,700 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111114c298fbe9db47a0baea89d238a6913a_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111114c298fbe9db47a0baea89d238a6913a_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:37,700 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/d4471c0d03854f1fa77fdea30bd27342, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:37,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/d4471c0d03854f1fa77fdea30bd27342 is 175, key is test_row_0/A:col10/1731296495751/Put/seqid=0 2024-11-11T03:41:37,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742224_1400 (size=39749) 2024-11-11T03:41:37,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296557722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:37,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T03:41:37,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:37,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:37,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296557912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:37,921 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:37,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296557922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:37,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296557922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:37,928 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,928 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296557932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:37,941 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:37,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:37,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:37,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T03:41:38,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296558042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:38,105 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/d4471c0d03854f1fa77fdea30bd27342 2024-11-11T03:41:38,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2e45fda3332e4197b6ab3e619c0d93d6 is 50, key is test_row_0/B:col10/1731296495751/Put/seqid=0 2024-11-11T03:41:38,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742225_1401 (size=12151) 2024-11-11T03:41:38,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T03:41:38,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2e45fda3332e4197b6ab3e619c0d93d6 2024-11-11T03:41:38,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/b3e51bbb619347a88795fc5f08ecdee4 is 50, key is test_row_0/C:col10/1731296495751/Put/seqid=0 2024-11-11T03:41:38,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742226_1402 (size=12151) 2024-11-11T03:41:38,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296558562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:38,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:38,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:38,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:38,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:38,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T03:41:38,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:38,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/b3e51bbb619347a88795fc5f08ecdee4 2024-11-11T03:41:38,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/d4471c0d03854f1fa77fdea30bd27342 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/d4471c0d03854f1fa77fdea30bd27342 2024-11-11T03:41:38,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/d4471c0d03854f1fa77fdea30bd27342, entries=200, sequenceid=208, filesize=38.8 K 2024-11-11T03:41:38,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/2e45fda3332e4197b6ab3e619c0d93d6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e45fda3332e4197b6ab3e619c0d93d6 2024-11-11T03:41:38,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e45fda3332e4197b6ab3e619c0d93d6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T03:41:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/b3e51bbb619347a88795fc5f08ecdee4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3e51bbb619347a88795fc5f08ecdee4 2024-11-11T03:41:38,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3e51bbb619347a88795fc5f08ecdee4, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T03:41:38,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4216aa280844bc244aeea907e086297f in 1657ms, sequenceid=208, compaction requested=true 2024-11-11T03:41:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:38,945 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:41:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:38,945 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:41:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/B is initiating minor compaction (all files) 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/A is initiating minor compaction (all files) 2024-11-11T03:41:38,946 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/B in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,946 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/A in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,946 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/8994d9dcf4134a3d8ccba864218c76ee, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e1f62f8171c4149934eaa46f9a1e6ca, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a1334fbb18af4f439c10db98fa30a957, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e45fda3332e4197b6ab3e619c0d93d6] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=47.8 K 2024-11-11T03:41:38,946 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/fed79a879fec482dadf51a507866dcc0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/69d3496271d14cd7b4c45f09ff4af215, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/97096e41f11f4098a61408429a36ba87, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/d4471c0d03854f1fa77fdea30bd27342] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=130.3 K 2024-11-11T03:41:38,946 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/fed79a879fec482dadf51a507866dcc0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/69d3496271d14cd7b4c45f09ff4af215, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/97096e41f11f4098a61408429a36ba87, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/d4471c0d03854f1fa77fdea30bd27342] 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8994d9dcf4134a3d8ccba864218c76ee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731296490691 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed79a879fec482dadf51a507866dcc0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731296490691 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1f62f8171c4149934eaa46f9a1e6ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731296492890 2024-11-11T03:41:38,946 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d3496271d14cd7b4c45f09ff4af215, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731296492890 2024-11-11T03:41:38,947 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting a1334fbb18af4f439c10db98fa30a957, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731296495078 2024-11-11T03:41:38,947 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97096e41f11f4098a61408429a36ba87, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731296495078 2024-11-11T03:41:38,947 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e45fda3332e4197b6ab3e619c0d93d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731296495728 2024-11-11T03:41:38,947 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4471c0d03854f1fa77fdea30bd27342, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731296495728 2024-11-11T03:41:38,954 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:38,955 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#B#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:38,956 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/f9d7a7f5ef3449b591f9b6032b9023ca is 50, key is test_row_0/B:col10/1731296495751/Put/seqid=0 2024-11-11T03:41:38,959 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111193d23651ee347db827f59344e279439_4216aa280844bc244aeea907e086297f store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:38,961 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111193d23651ee347db827f59344e279439_4216aa280844bc244aeea907e086297f, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:38,962 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111193d23651ee347db827f59344e279439_4216aa280844bc244aeea907e086297f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:38,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742228_1404 (size=12629) 2024-11-11T03:41:38,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742227_1403 (size=4469) 2024-11-11T03:41:38,973 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/f9d7a7f5ef3449b591f9b6032b9023ca as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/f9d7a7f5ef3449b591f9b6032b9023ca 2024-11-11T03:41:38,977 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4216aa280844bc244aeea907e086297f/B of 4216aa280844bc244aeea907e086297f into f9d7a7f5ef3449b591f9b6032b9023ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:38,977 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:38,977 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/B, priority=12, startTime=1731296498945; duration=0sec 2024-11-11T03:41:38,977 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:38,977 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:B 2024-11-11T03:41:38,977 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:41:38,979 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:41:38,979 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/C is initiating minor compaction (all files) 2024-11-11T03:41:38,979 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/C in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:38,979 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/f86a62e2eccf4f4591adb9a9e98b7787, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/2ae7a5396ad74da8a237f7cce77916ee, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/1a31c55f98f640f0871257713606520b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3e51bbb619347a88795fc5f08ecdee4] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=47.8 K 2024-11-11T03:41:38,979 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f86a62e2eccf4f4591adb9a9e98b7787, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731296490691 2024-11-11T03:41:38,979 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae7a5396ad74da8a237f7cce77916ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731296492890 2024-11-11T03:41:38,980 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a31c55f98f640f0871257713606520b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731296495078 2024-11-11T03:41:38,980 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e51bbb619347a88795fc5f08ecdee4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731296495728 2024-11-11T03:41:38,989 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#C#compaction#340 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:38,989 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/eb2ef9037498485f8dad010b60530d23 is 50, key is test_row_0/C:col10/1731296495751/Put/seqid=0 2024-11-11T03:41:38,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742229_1405 (size=12629) 2024-11-11T03:41:39,001 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/eb2ef9037498485f8dad010b60530d23 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/eb2ef9037498485f8dad010b60530d23 2024-11-11T03:41:39,004 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4216aa280844bc244aeea907e086297f/C of 4216aa280844bc244aeea907e086297f into eb2ef9037498485f8dad010b60530d23(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:39,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:39,004 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/C, priority=12, startTime=1731296498945; duration=0sec 2024-11-11T03:41:39,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:39,004 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:C 2024-11-11T03:41:39,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T03:41:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:39,068 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T03:41:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111101395dbace8f4962a2f273ac4d8d2a70_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296497384/Put/seqid=0 2024-11-11T03:41:39,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742230_1406 (size=12304) 2024-11-11T03:41:39,372 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:39,372 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/e1dc38fa7ab74383937eab53c76c62cf is 175, key is test_row_0/A:col10/1731296495751/Put/seqid=0 2024-11-11T03:41:39,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742231_1407 (size=31583) 2024-11-11T03:41:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:39,479 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111101395dbace8f4962a2f273ac4d8d2a70_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111101395dbace8f4962a2f273ac4d8d2a70_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/4fed5ded07bb4eeeb5d0972c5c97d974, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/4fed5ded07bb4eeeb5d0972c5c97d974 is 175, key is test_row_0/A:col10/1731296497384/Put/seqid=0 2024-11-11T03:41:39,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742232_1408 (size=31105) 2024-11-11T03:41:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:39,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:39,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296559678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:39,778 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/e1dc38fa7ab74383937eab53c76c62cf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/e1dc38fa7ab74383937eab53c76c62cf 2024-11-11T03:41:39,781 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4216aa280844bc244aeea907e086297f/A of 4216aa280844bc244aeea907e086297f into e1dc38fa7ab74383937eab53c76c62cf(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:39,781 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:39,781 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/A, priority=12, startTime=1731296498945; duration=0sec 2024-11-11T03:41:39,782 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:39,782 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:A 2024-11-11T03:41:39,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296559791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T03:41:39,888 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/4fed5ded07bb4eeeb5d0972c5c97d974 2024-11-11T03:41:39,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/d37e01500f564f3ea38f4bc7ebc07a7b is 50, key is test_row_0/B:col10/1731296497384/Put/seqid=0 2024-11-11T03:41:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742233_1409 (size=12151) 2024-11-11T03:41:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43858 deadline: 1731296559942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:39,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43832 deadline: 1731296559942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:39,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43866 deadline: 1731296559942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:39,947 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:39,947 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:39,947 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_1' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:39,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43828 deadline: 1731296559961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:39,969 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in 4216aa280844bc244aeea907e086297f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296560002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:40,298 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/d37e01500f564f3ea38f4bc7ebc07a7b 2024-11-11T03:41:40,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/b3110c635e8d41a38f49560cc0a3604f is 50, key is test_row_0/C:col10/1731296497384/Put/seqid=0 2024-11-11T03:41:40,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742234_1410 (size=12151) 2024-11-11T03:41:40,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296560312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:40,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/b3110c635e8d41a38f49560cc0a3604f 2024-11-11T03:41:40,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/4fed5ded07bb4eeeb5d0972c5c97d974 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/4fed5ded07bb4eeeb5d0972c5c97d974 2024-11-11T03:41:40,713 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/4fed5ded07bb4eeeb5d0972c5c97d974, entries=150, sequenceid=226, filesize=30.4 K 2024-11-11T03:41:40,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/d37e01500f564f3ea38f4bc7ebc07a7b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/d37e01500f564f3ea38f4bc7ebc07a7b 2024-11-11T03:41:40,716 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/d37e01500f564f3ea38f4bc7ebc07a7b, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T03:41:40,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/b3110c635e8d41a38f49560cc0a3604f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3110c635e8d41a38f49560cc0a3604f 2024-11-11T03:41:40,720 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3110c635e8d41a38f49560cc0a3604f, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T03:41:40,720 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4216aa280844bc244aeea907e086297f in 1652ms, sequenceid=226, compaction requested=false 2024-11-11T03:41:40,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:40,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:40,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T03:41:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T03:41:40,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T03:41:40,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0270 sec 2024-11-11T03:41:40,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.0310 sec 2024-11-11T03:41:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:40,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:41:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:40,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d322c6b504a54239a2a085c8972d8655_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296499626/Put/seqid=0 2024-11-11T03:41:40,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742235_1411 (size=14794) 2024-11-11T03:41:40,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296560894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:41,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296561011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:41,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:41,237 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d322c6b504a54239a2a085c8972d8655_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d322c6b504a54239a2a085c8972d8655_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:41,238 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/8d681924f3c842a682fc76d6f9528436, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:41,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/8d681924f3c842a682fc76d6f9528436 is 175, key is test_row_0/A:col10/1731296499626/Put/seqid=0 2024-11-11T03:41:41,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296561232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:41,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742236_1412 (size=39749) 2024-11-11T03:41:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296561551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:41,642 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/8d681924f3c842a682fc76d6f9528436 2024-11-11T03:41:41,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a23c61375abe444b81cdca6433c02062 is 50, key is test_row_0/B:col10/1731296499626/Put/seqid=0 2024-11-11T03:41:41,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742237_1413 (size=12151) 2024-11-11T03:41:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T03:41:41,852 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:41,852 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T03:41:41,854 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T03:41:41,854 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:41,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T03:41:42,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T03:41:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a23c61375abe444b81cdca6433c02062 2024-11-11T03:41:42,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/08e29c6d25e44acfaa542cf2b1405cff is 50, key is test_row_0/C:col10/1731296499626/Put/seqid=0 2024-11-11T03:41:42,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742238_1414 (size=12151) 2024-11-11T03:41:42,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43856 deadline: 1731296562061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:41:42,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T03:41:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T03:41:42,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T03:41:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/08e29c6d25e44acfaa542cf2b1405cff 2024-11-11T03:41:42,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/8d681924f3c842a682fc76d6f9528436 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/8d681924f3c842a682fc76d6f9528436 2024-11-11T03:41:42,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T03:41:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:42,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/8d681924f3c842a682fc76d6f9528436, entries=200, sequenceid=244, filesize=38.8 K 2024-11-11T03:41:42,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/a23c61375abe444b81cdca6433c02062 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a23c61375abe444b81cdca6433c02062 2024-11-11T03:41:42,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a23c61375abe444b81cdca6433c02062, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T03:41:42,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/08e29c6d25e44acfaa542cf2b1405cff as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/08e29c6d25e44acfaa542cf2b1405cff 2024-11-11T03:41:42,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/08e29c6d25e44acfaa542cf2b1405cff, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T03:41:42,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4216aa280844bc244aeea907e086297f in 1650ms, sequenceid=244, compaction requested=true 2024-11-11T03:41:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:42,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:42,476 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4216aa280844bc244aeea907e086297f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:42,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:42,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:42,476 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:42,476 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/B is initiating minor compaction (all files) 2024-11-11T03:41:42,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/A is initiating minor compaction (all files) 2024-11-11T03:41:42,477 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/B in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,477 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/A in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,477 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/f9d7a7f5ef3449b591f9b6032b9023ca, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/d37e01500f564f3ea38f4bc7ebc07a7b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a23c61375abe444b81cdca6433c02062] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=36.1 K 2024-11-11T03:41:42,477 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/e1dc38fa7ab74383937eab53c76c62cf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/4fed5ded07bb4eeeb5d0972c5c97d974, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/8d681924f3c842a682fc76d6f9528436] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=100.0 K 2024-11-11T03:41:42,477 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,477 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/e1dc38fa7ab74383937eab53c76c62cf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/4fed5ded07bb4eeeb5d0972c5c97d974, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/8d681924f3c842a682fc76d6f9528436] 2024-11-11T03:41:42,477 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1dc38fa7ab74383937eab53c76c62cf, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731296495728 2024-11-11T03:41:42,477 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d7a7f5ef3449b591f9b6032b9023ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731296495728 2024-11-11T03:41:42,477 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d37e01500f564f3ea38f4bc7ebc07a7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296497350 2024-11-11T03:41:42,477 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fed5ded07bb4eeeb5d0972c5c97d974, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296497350 2024-11-11T03:41:42,478 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting a23c61375abe444b81cdca6433c02062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731296499626 2024-11-11T03:41:42,478 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d681924f3c842a682fc76d6f9528436, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731296499626 2024-11-11T03:41:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T03:41:42,484 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#B#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:42,484 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/1fefae64595e4d13b329b8d4f4265782 is 50, key is test_row_0/B:col10/1731296499626/Put/seqid=0 2024-11-11T03:41:42,485 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742239_1415 (size=12731) 2024-11-11T03:41:42,495 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111493674181a0e4801babed2dcdea8029c_4216aa280844bc244aeea907e086297f store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:42,496 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111493674181a0e4801babed2dcdea8029c_4216aa280844bc244aeea907e086297f, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:42,496 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111493674181a0e4801babed2dcdea8029c_4216aa280844bc244aeea907e086297f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:42,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742240_1416 (size=4469) 2024-11-11T03:41:42,570 INFO [Thread-1587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1587. 2024-11-11T03:41:42,570 DEBUG [Thread-1587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:42,570 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,570 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:42,570 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-11T03:41:42,571 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:42,571 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,571 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:42,571 INFO [Thread-1589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1589. 2024-11-11T03:41:42,571 DEBUG [Thread-1589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:42,571 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,571 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:42,581 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-11T03:41:42,581 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-11T03:41:42,581 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:42,581 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:42,581 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,581 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,581 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,581 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:42,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:42,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:42,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T03:41:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,616 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:41:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:42,621 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:41:42,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111226b4d3130ca4384a8d515f83b1be110_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296500881/Put/seqid=0 2024-11-11T03:41:42,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742241_1417 (size=12354) 2024-11-11T03:41:42,899 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/1fefae64595e4d13b329b8d4f4265782 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/1fefae64595e4d13b329b8d4f4265782 2024-11-11T03:41:42,901 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:42,902 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/19a425e6404b428c937a5b8921b26401 is 175, key is test_row_0/A:col10/1731296499626/Put/seqid=0 2024-11-11T03:41:42,904 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/B of 4216aa280844bc244aeea907e086297f into 1fefae64595e4d13b329b8d4f4265782(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:42,904 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:42,904 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/B, priority=13, startTime=1731296502476; duration=0sec 2024-11-11T03:41:42,905 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:42,905 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:B 2024-11-11T03:41:42,905 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:42,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742242_1418 (size=31685) 2024-11-11T03:41:42,906 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:42,906 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): 4216aa280844bc244aeea907e086297f/C is initiating minor compaction (all files) 2024-11-11T03:41:42,906 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4216aa280844bc244aeea907e086297f/C in TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:42,906 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/eb2ef9037498485f8dad010b60530d23, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3110c635e8d41a38f49560cc0a3604f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/08e29c6d25e44acfaa542cf2b1405cff] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp, totalSize=36.1 K 2024-11-11T03:41:42,907 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting eb2ef9037498485f8dad010b60530d23, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731296495728 2024-11-11T03:41:42,907 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b3110c635e8d41a38f49560cc0a3604f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731296497350 2024-11-11T03:41:42,907 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e29c6d25e44acfaa542cf2b1405cff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731296499626 2024-11-11T03:41:42,915 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4216aa280844bc244aeea907e086297f#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:42,915 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/d15dfefc9a26448b9c176c8c77dd585a is 50, key is test_row_0/C:col10/1731296499626/Put/seqid=0 2024-11-11T03:41:42,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742243_1419 (size=12731) 2024-11-11T03:41:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T03:41:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:43,033 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111226b4d3130ca4384a8d515f83b1be110_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111226b4d3130ca4384a8d515f83b1be110_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:43,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/03dfbb84eae943abbbfa55693569d3b1, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:43,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/03dfbb84eae943abbbfa55693569d3b1 is 175, key is test_row_0/A:col10/1731296500881/Put/seqid=0 2024-11-11T03:41:43,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742244_1420 (size=31155) 2024-11-11T03:41:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:43,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. as already flushing 2024-11-11T03:41:43,076 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-11T03:41:43,076 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:43,076 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,077 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:43,314 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/19a425e6404b428c937a5b8921b26401 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/19a425e6404b428c937a5b8921b26401 2024-11-11T03:41:43,322 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/A of 4216aa280844bc244aeea907e086297f into 19a425e6404b428c937a5b8921b26401(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:43,322 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:43,322 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/A, priority=13, startTime=1731296502476; duration=0sec 2024-11-11T03:41:43,322 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:43,322 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:A 2024-11-11T03:41:43,325 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/d15dfefc9a26448b9c176c8c77dd585a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d15dfefc9a26448b9c176c8c77dd585a 2024-11-11T03:41:43,332 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4216aa280844bc244aeea907e086297f/C of 4216aa280844bc244aeea907e086297f into d15dfefc9a26448b9c176c8c77dd585a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:43,332 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:43,332 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., storeName=4216aa280844bc244aeea907e086297f/C, priority=13, startTime=1731296502476; duration=0sec 2024-11-11T03:41:43,332 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:43,332 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4216aa280844bc244aeea907e086297f:C 2024-11-11T03:41:43,441 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/03dfbb84eae943abbbfa55693569d3b1 2024-11-11T03:41:43,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/46d6900f278e45a29d08436e0f782cb9 is 50, key is test_row_0/B:col10/1731296500881/Put/seqid=0 2024-11-11T03:41:43,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742245_1421 (size=12201) 2024-11-11T03:41:43,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/46d6900f278e45a29d08436e0f782cb9 2024-11-11T03:41:43,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/c5634eaf90524df79b45f1938ce4b667 is 50, key is test_row_0/C:col10/1731296500881/Put/seqid=0 2024-11-11T03:41:43,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742246_1422 (size=12201) 2024-11-11T03:41:43,976 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-11T03:41:43,976 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-11T03:41:43,977 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:43,977 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:43,977 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,977 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,977 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,977 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:43,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:43,985 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-11T03:41:43,985 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:43,985 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,985 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:43,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T03:41:44,014 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-11T03:41:44,015 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:41:44,015 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:44,015 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:44,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:44,324 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/c5634eaf90524df79b45f1938ce4b667 2024-11-11T03:41:44,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/03dfbb84eae943abbbfa55693569d3b1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/03dfbb84eae943abbbfa55693569d3b1 2024-11-11T03:41:44,335 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/03dfbb84eae943abbbfa55693569d3b1, entries=150, sequenceid=259, filesize=30.4 K 2024-11-11T03:41:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/46d6900f278e45a29d08436e0f782cb9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/46d6900f278e45a29d08436e0f782cb9 2024-11-11T03:41:44,342 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/46d6900f278e45a29d08436e0f782cb9, entries=150, sequenceid=259, filesize=11.9 K 2024-11-11T03:41:44,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/c5634eaf90524df79b45f1938ce4b667 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/c5634eaf90524df79b45f1938ce4b667 2024-11-11T03:41:44,346 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/c5634eaf90524df79b45f1938ce4b667, entries=150, sequenceid=259, filesize=11.9 K 2024-11-11T03:41:44,347 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=33.54 KB/34350 for 4216aa280844bc244aeea907e086297f in 1731ms, sequenceid=259, compaction requested=false 2024-11-11T03:41:44,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 4216aa280844bc244aeea907e086297f: 2024-11-11T03:41:44,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:44,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T03:41:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T03:41:44,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T03:41:44,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4940 sec 2024-11-11T03:41:44,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.4980 sec 2024-11-11T03:41:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T03:41:46,023 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1160 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3470 rows 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1164 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3488 rows 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1162 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3476 rows 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1159 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3476 rows 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1153 2024-11-11T03:41:46,024 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3459 rows 2024-11-11T03:41:46,025 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:41:46,025 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:41:46,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:46,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:46,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:41:46,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T03:41:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T03:41:46,033 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296506033"}]},"ts":"1731296506033"} 2024-11-11T03:41:46,035 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T03:41:46,035 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T03:41:46,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:41:46,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, UNASSIGN}] 2024-11-11T03:41:46,039 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, UNASSIGN 2024-11-11T03:41:46,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:46,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, UNASSIGN because future has completed 2024-11-11T03:41:46,042 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:41:46,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:41:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T03:41:46,196 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:46,196 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:41:46,196 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 4216aa280844bc244aeea907e086297f, disabling compactions & flushes 2024-11-11T03:41:46,196 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:46,196 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:46,196 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. after waiting 0 ms 2024-11-11T03:41:46,196 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:46,197 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 4216aa280844bc244aeea907e086297f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T03:41:46,197 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=A 2024-11-11T03:41:46,197 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:46,197 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=B 2024-11-11T03:41:46,197 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:46,198 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4216aa280844bc244aeea907e086297f, store=C 2024-11-11T03:41:46,198 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:46,207 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0da4f0174c440e2b66a7b1759a82c89_4216aa280844bc244aeea907e086297f is 50, key is test_row_0/A:col10/1731296504012/Put/seqid=0 2024-11-11T03:41:46,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742247_1423 (size=12454) 2024-11-11T03:41:46,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/a66785fd89874bc88ce16688492c7581, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/12baad22401d41faac6ef8d08f6ae46f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/408f0acc985441318b9bf7265b09dc09, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/37542e8faf4e4b97a99c40fff5307639, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/265441ce8b064957be54b97d81f9ddb4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/2a628c02ea594c838807f2715390da06, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/6ecf1c6c26584bcd8d74ab1d9d079e5f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/de047e877413475b87e4bdca90b2b04b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/7c073d3a8bdd4b1b968ccd7af2bb7d44, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/23a906e7715b436ea56dd6e3262c0276, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/fed79a879fec482dadf51a507866dcc0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/69d3496271d14cd7b4c45f09ff4af215, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/97096e41f11f4098a61408429a36ba87, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/d4471c0d03854f1fa77fdea30bd27342, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/e1dc38fa7ab74383937eab53c76c62cf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/4fed5ded07bb4eeeb5d0972c5c97d974, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/8d681924f3c842a682fc76d6f9528436] to archive 2024-11-11T03:41:46,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:41:46,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/a66785fd89874bc88ce16688492c7581 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/a66785fd89874bc88ce16688492c7581 2024-11-11T03:41:46,285 WARN [regionserver/3ffe7f9d089e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-11T03:41:46,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/12baad22401d41faac6ef8d08f6ae46f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/12baad22401d41faac6ef8d08f6ae46f 2024-11-11T03:41:46,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/408f0acc985441318b9bf7265b09dc09 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/408f0acc985441318b9bf7265b09dc09 2024-11-11T03:41:46,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/37542e8faf4e4b97a99c40fff5307639 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/37542e8faf4e4b97a99c40fff5307639 2024-11-11T03:41:46,289 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/265441ce8b064957be54b97d81f9ddb4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/265441ce8b064957be54b97d81f9ddb4 2024-11-11T03:41:46,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/2a628c02ea594c838807f2715390da06 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/2a628c02ea594c838807f2715390da06 2024-11-11T03:41:46,291 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/6ecf1c6c26584bcd8d74ab1d9d079e5f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/6ecf1c6c26584bcd8d74ab1d9d079e5f 2024-11-11T03:41:46,292 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/de047e877413475b87e4bdca90b2b04b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/de047e877413475b87e4bdca90b2b04b 2024-11-11T03:41:46,293 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/7c073d3a8bdd4b1b968ccd7af2bb7d44 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/7c073d3a8bdd4b1b968ccd7af2bb7d44 2024-11-11T03:41:46,294 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/23a906e7715b436ea56dd6e3262c0276 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/23a906e7715b436ea56dd6e3262c0276 2024-11-11T03:41:46,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/fed79a879fec482dadf51a507866dcc0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/fed79a879fec482dadf51a507866dcc0 2024-11-11T03:41:46,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/69d3496271d14cd7b4c45f09ff4af215 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/69d3496271d14cd7b4c45f09ff4af215 2024-11-11T03:41:46,296 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/97096e41f11f4098a61408429a36ba87 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/97096e41f11f4098a61408429a36ba87 2024-11-11T03:41:46,297 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/d4471c0d03854f1fa77fdea30bd27342 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/d4471c0d03854f1fa77fdea30bd27342 2024-11-11T03:41:46,297 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/e1dc38fa7ab74383937eab53c76c62cf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/e1dc38fa7ab74383937eab53c76c62cf 2024-11-11T03:41:46,298 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/4fed5ded07bb4eeeb5d0972c5c97d974 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/4fed5ded07bb4eeeb5d0972c5c97d974 2024-11-11T03:41:46,299 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/8d681924f3c842a682fc76d6f9528436 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/8d681924f3c842a682fc76d6f9528436 2024-11-11T03:41:46,301 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/43c7cdb822cb44a8b7b7e0536ab60c76, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ce4313f96edf41fea72c91456c34447c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/5d7ae6e21e694f2d948f0e08c028da73, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/40382abc2afa4960be3777e65bbf229c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2a43071baa994a5aa72ccb17535c04ab, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/03c8f6d328084076a93a8546a49c5f32, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/b9ea154364a64913bb9169449c8d1487, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a15963514d8741798a2fc6c532091b8e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ba5a64af373e4a6282113f59a7dac79a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/8994d9dcf4134a3d8ccba864218c76ee, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/cbd4c05c46404c40baec4cf11413574d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e1f62f8171c4149934eaa46f9a1e6ca, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a1334fbb18af4f439c10db98fa30a957, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/f9d7a7f5ef3449b591f9b6032b9023ca, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e45fda3332e4197b6ab3e619c0d93d6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/d37e01500f564f3ea38f4bc7ebc07a7b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a23c61375abe444b81cdca6433c02062] to archive 2024-11-11T03:41:46,301 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:41:46,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/43c7cdb822cb44a8b7b7e0536ab60c76 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/43c7cdb822cb44a8b7b7e0536ab60c76 2024-11-11T03:41:46,303 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ce4313f96edf41fea72c91456c34447c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ce4313f96edf41fea72c91456c34447c 2024-11-11T03:41:46,304 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/5d7ae6e21e694f2d948f0e08c028da73 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/5d7ae6e21e694f2d948f0e08c028da73 2024-11-11T03:41:46,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/40382abc2afa4960be3777e65bbf229c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/40382abc2afa4960be3777e65bbf229c 2024-11-11T03:41:46,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2a43071baa994a5aa72ccb17535c04ab to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2a43071baa994a5aa72ccb17535c04ab 2024-11-11T03:41:46,306 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/03c8f6d328084076a93a8546a49c5f32 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/03c8f6d328084076a93a8546a49c5f32 2024-11-11T03:41:46,307 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/b9ea154364a64913bb9169449c8d1487 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/b9ea154364a64913bb9169449c8d1487 2024-11-11T03:41:46,307 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a15963514d8741798a2fc6c532091b8e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a15963514d8741798a2fc6c532091b8e 2024-11-11T03:41:46,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ba5a64af373e4a6282113f59a7dac79a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/ba5a64af373e4a6282113f59a7dac79a 2024-11-11T03:41:46,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/8994d9dcf4134a3d8ccba864218c76ee to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/8994d9dcf4134a3d8ccba864218c76ee 2024-11-11T03:41:46,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/cbd4c05c46404c40baec4cf11413574d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/cbd4c05c46404c40baec4cf11413574d 2024-11-11T03:41:46,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e1f62f8171c4149934eaa46f9a1e6ca to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e1f62f8171c4149934eaa46f9a1e6ca 2024-11-11T03:41:46,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a1334fbb18af4f439c10db98fa30a957 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a1334fbb18af4f439c10db98fa30a957 2024-11-11T03:41:46,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/f9d7a7f5ef3449b591f9b6032b9023ca to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/f9d7a7f5ef3449b591f9b6032b9023ca 2024-11-11T03:41:46,312 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e45fda3332e4197b6ab3e619c0d93d6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/2e45fda3332e4197b6ab3e619c0d93d6 2024-11-11T03:41:46,312 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/d37e01500f564f3ea38f4bc7ebc07a7b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/d37e01500f564f3ea38f4bc7ebc07a7b 2024-11-11T03:41:46,313 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a23c61375abe444b81cdca6433c02062 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/a23c61375abe444b81cdca6433c02062 2024-11-11T03:41:46,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d3f44a7745024418be3e4daaa68217c4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/3228aa00299643a092810cfcd0eddc27, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e7b18eb547a64d9aae79f517b9db776c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/03da2b87b3b34a10a7bd67dc895f6b36, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/65ce31dc1bcb4d5abc6321ba7fa5894a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/503cd77b2dc24eb388e815dbdbb5a173, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/6afd3396342c423fae043ebdffa36ef1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/4f6a32f8379a42d88049b013c5e9f5c4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e6b6ffa172c74f3cb7135741bfecaea5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/f86a62e2eccf4f4591adb9a9e98b7787, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/00a0d25f404345429d8d1c6dfc90de7b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/2ae7a5396ad74da8a237f7cce77916ee, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/1a31c55f98f640f0871257713606520b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/eb2ef9037498485f8dad010b60530d23, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3e51bbb619347a88795fc5f08ecdee4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3110c635e8d41a38f49560cc0a3604f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/08e29c6d25e44acfaa542cf2b1405cff] to archive 2024-11-11T03:41:46,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:41:46,317 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d3f44a7745024418be3e4daaa68217c4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d3f44a7745024418be3e4daaa68217c4 2024-11-11T03:41:46,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/3228aa00299643a092810cfcd0eddc27 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/3228aa00299643a092810cfcd0eddc27 2024-11-11T03:41:46,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e7b18eb547a64d9aae79f517b9db776c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e7b18eb547a64d9aae79f517b9db776c 2024-11-11T03:41:46,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/03da2b87b3b34a10a7bd67dc895f6b36 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/03da2b87b3b34a10a7bd67dc895f6b36 2024-11-11T03:41:46,320 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/65ce31dc1bcb4d5abc6321ba7fa5894a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/65ce31dc1bcb4d5abc6321ba7fa5894a 2024-11-11T03:41:46,320 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/503cd77b2dc24eb388e815dbdbb5a173 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/503cd77b2dc24eb388e815dbdbb5a173 2024-11-11T03:41:46,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/6afd3396342c423fae043ebdffa36ef1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/6afd3396342c423fae043ebdffa36ef1 2024-11-11T03:41:46,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/4f6a32f8379a42d88049b013c5e9f5c4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/4f6a32f8379a42d88049b013c5e9f5c4 2024-11-11T03:41:46,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e6b6ffa172c74f3cb7135741bfecaea5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e6b6ffa172c74f3cb7135741bfecaea5 2024-11-11T03:41:46,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/f86a62e2eccf4f4591adb9a9e98b7787 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/f86a62e2eccf4f4591adb9a9e98b7787 2024-11-11T03:41:46,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/00a0d25f404345429d8d1c6dfc90de7b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/00a0d25f404345429d8d1c6dfc90de7b 2024-11-11T03:41:46,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/2ae7a5396ad74da8a237f7cce77916ee to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/2ae7a5396ad74da8a237f7cce77916ee 2024-11-11T03:41:46,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/1a31c55f98f640f0871257713606520b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/1a31c55f98f640f0871257713606520b 2024-11-11T03:41:46,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/eb2ef9037498485f8dad010b60530d23 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/eb2ef9037498485f8dad010b60530d23 2024-11-11T03:41:46,327 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3e51bbb619347a88795fc5f08ecdee4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3e51bbb619347a88795fc5f08ecdee4 2024-11-11T03:41:46,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3110c635e8d41a38f49560cc0a3604f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/b3110c635e8d41a38f49560cc0a3604f 2024-11-11T03:41:46,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/08e29c6d25e44acfaa542cf2b1405cff to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/08e29c6d25e44acfaa542cf2b1405cff 2024-11-11T03:41:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T03:41:46,613 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:46,622 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0da4f0174c440e2b66a7b1759a82c89_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0da4f0174c440e2b66a7b1759a82c89_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:46,623 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/705872bc3e524dbdbd4c186a839d61fd, store: [table=TestAcidGuarantees family=A region=4216aa280844bc244aeea907e086297f] 2024-11-11T03:41:46,624 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/705872bc3e524dbdbd4c186a839d61fd is 175, key is test_row_0/A:col10/1731296504012/Put/seqid=0 2024-11-11T03:41:46,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742248_1424 (size=31255) 2024-11-11T03:41:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T03:41:47,029 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/705872bc3e524dbdbd4c186a839d61fd 2024-11-11T03:41:47,044 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/49b1a438df1745b6a718da9a225618e7 is 50, key is test_row_0/B:col10/1731296504012/Put/seqid=0 2024-11-11T03:41:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742249_1425 (size=12301) 2024-11-11T03:41:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T03:41:47,448 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/49b1a438df1745b6a718da9a225618e7 2024-11-11T03:41:47,462 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e135e13033bc4fb08a5fa7242d1d6583 is 50, key is test_row_0/C:col10/1731296504012/Put/seqid=0 2024-11-11T03:41:47,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742250_1426 (size=12301) 2024-11-11T03:41:47,867 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e135e13033bc4fb08a5fa7242d1d6583 2024-11-11T03:41:47,877 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/A/705872bc3e524dbdbd4c186a839d61fd as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/705872bc3e524dbdbd4c186a839d61fd 2024-11-11T03:41:47,882 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/705872bc3e524dbdbd4c186a839d61fd, entries=150, sequenceid=270, filesize=30.5 K 2024-11-11T03:41:47,884 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/B/49b1a438df1745b6a718da9a225618e7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/49b1a438df1745b6a718da9a225618e7 2024-11-11T03:41:47,888 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/49b1a438df1745b6a718da9a225618e7, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T03:41:47,888 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/.tmp/C/e135e13033bc4fb08a5fa7242d1d6583 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e135e13033bc4fb08a5fa7242d1d6583 2024-11-11T03:41:47,892 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e135e13033bc4fb08a5fa7242d1d6583, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T03:41:47,892 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4216aa280844bc244aeea907e086297f in 1696ms, sequenceid=270, compaction requested=true 2024-11-11T03:41:47,897 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/recovered.edits/273.seqid, newMaxSeqId=273, maxSeqId=4 2024-11-11T03:41:47,897 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. 2024-11-11T03:41:47,897 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 4216aa280844bc244aeea907e086297f: Waiting for close lock at 1731296506196Running coprocessor pre-close hooks at 1731296506196Disabling compacts and flushes for region at 1731296506196Disabling writes for close at 1731296506196Obtaining lock to block concurrent updates at 1731296506197 (+1 ms)Preparing flush snapshotting stores in 4216aa280844bc244aeea907e086297f at 1731296506197Finished memstore snapshotting TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731296506198 (+1 ms)Flushing stores of TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f. at 1731296506199 (+1 ms)Flushing 4216aa280844bc244aeea907e086297f/A: creating writer at 1731296506199Flushing mob file 4216aa280844bc244aeea907e086297f/A: appending metadata at 1731296506207 (+8 ms)Flushing mob file 4216aa280844bc244aeea907e086297f/A: closing flushed file at 1731296506207Flushing 4216aa280844bc244aeea907e086297f/A: appending metadata at 1731296506623 (+416 ms)Flushing 4216aa280844bc244aeea907e086297f/A: closing flushed file at 1731296506623Flushing 4216aa280844bc244aeea907e086297f/B: creating writer at 1731296507039 (+416 ms)Flushing 4216aa280844bc244aeea907e086297f/B: appending metadata at 1731296507043 (+4 ms)Flushing 4216aa280844bc244aeea907e086297f/B: closing flushed file at 1731296507043Flushing 4216aa280844bc244aeea907e086297f/C: creating writer at 1731296507457 (+414 ms)Flushing 4216aa280844bc244aeea907e086297f/C: appending metadata at 1731296507462 (+5 ms)Flushing 4216aa280844bc244aeea907e086297f/C: closing flushed file at 1731296507462Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a6bd0ec: reopening flushed file at 1731296507876 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22d89002: reopening flushed file at 1731296507883 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dfd81eb: reopening flushed file at 1731296507888 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4216aa280844bc244aeea907e086297f in 1696ms, sequenceid=270, compaction requested=true at 1731296507892 (+4 ms)Writing region close event to WAL at 1731296507894 (+2 ms)Running coprocessor post-close hooks at 1731296507897 (+3 ms)Closed at 1731296507897 2024-11-11T03:41:47,899 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 4216aa280844bc244aeea907e086297f 2024-11-11T03:41:47,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=4216aa280844bc244aeea907e086297f, regionState=CLOSED 2024-11-11T03:41:47,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:41:47,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T03:41:47,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 4216aa280844bc244aeea907e086297f, server=3ffe7f9d089e,34473,1731296384288 in 1.8590 sec 2024-11-11T03:41:47,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-11T03:41:47,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4216aa280844bc244aeea907e086297f, UNASSIGN in 1.8650 sec 2024-11-11T03:41:47,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T03:41:47,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8690 sec 2024-11-11T03:41:47,908 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296507908"}]},"ts":"1731296507908"} 2024-11-11T03:41:47,909 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T03:41:47,909 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T03:41:47,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-11T03:41:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T03:41:48,193 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:48,195 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T03:41:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:48,200 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T03:41:48,202 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:48,205 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,207 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/recovered.edits] 2024-11-11T03:41:48,210 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/03dfbb84eae943abbbfa55693569d3b1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/03dfbb84eae943abbbfa55693569d3b1 2024-11-11T03:41:48,211 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/19a425e6404b428c937a5b8921b26401 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/19a425e6404b428c937a5b8921b26401 2024-11-11T03:41:48,212 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/705872bc3e524dbdbd4c186a839d61fd to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/A/705872bc3e524dbdbd4c186a839d61fd 2024-11-11T03:41:48,214 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/1fefae64595e4d13b329b8d4f4265782 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/1fefae64595e4d13b329b8d4f4265782 2024-11-11T03:41:48,215 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/46d6900f278e45a29d08436e0f782cb9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/46d6900f278e45a29d08436e0f782cb9 2024-11-11T03:41:48,216 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/49b1a438df1745b6a718da9a225618e7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/B/49b1a438df1745b6a718da9a225618e7 2024-11-11T03:41:48,217 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/c5634eaf90524df79b45f1938ce4b667 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/c5634eaf90524df79b45f1938ce4b667 2024-11-11T03:41:48,218 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d15dfefc9a26448b9c176c8c77dd585a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/d15dfefc9a26448b9c176c8c77dd585a 2024-11-11T03:41:48,219 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e135e13033bc4fb08a5fa7242d1d6583 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/C/e135e13033bc4fb08a5fa7242d1d6583 2024-11-11T03:41:48,220 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/recovered.edits/273.seqid to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f/recovered.edits/273.seqid 2024-11-11T03:41:48,221 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,221 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T03:41:48,221 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T03:41:48,221 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T03:41:48,223 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111101395dbace8f4962a2f273ac4d8d2a70_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111101395dbace8f4962a2f273ac4d8d2a70_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,224 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110736190975794244ba2c8bca3fa20cd8_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110736190975794244ba2c8bca3fa20cd8_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,225 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111114c298fbe9db47a0baea89d238a6913a_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111114c298fbe9db47a0baea89d238a6913a_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,226 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111226b4d3130ca4384a8d515f83b1be110_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111226b4d3130ca4384a8d515f83b1be110_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,226 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111143489b6698c64be8959e4ac1b7e1e26e_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111143489b6698c64be8959e4ac1b7e1e26e_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,227 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ee3a64cb6214f17aaee40cc31865e80_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ee3a64cb6214f17aaee40cc31865e80_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,228 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119534e6c7ae4e4734a14a695eed5d2243_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119534e6c7ae4e4734a14a695eed5d2243_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,228 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a471d5546faf4b539f93e0f07cbef086_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a471d5546faf4b539f93e0f07cbef086_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,229 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b5dfe3b8fe4e4ac5abccef39e9b880b4_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b5dfe3b8fe4e4ac5abccef39e9b880b4_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,230 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0da4f0174c440e2b66a7b1759a82c89_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0da4f0174c440e2b66a7b1759a82c89_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,231 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c77d986eebc34c1cac8a61693355ce0c_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c77d986eebc34c1cac8a61693355ce0c_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,231 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d322c6b504a54239a2a085c8972d8655_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d322c6b504a54239a2a085c8972d8655_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,232 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebcfa5a28efe4e4a8e5071034e6fce91_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ebcfa5a28efe4e4a8e5071034e6fce91_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,233 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ec5c5e7ba8074ca89d88565b980a7f05_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ec5c5e7ba8074ca89d88565b980a7f05_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,233 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fec67f4af81040ba8e5dcbd882f9114b_4216aa280844bc244aeea907e086297f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fec67f4af81040ba8e5dcbd882f9114b_4216aa280844bc244aeea907e086297f 2024-11-11T03:41:48,234 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T03:41:48,235 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:48,237 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T03:41:48,239 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T03:41:48,240 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:48,240 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T03:41:48,240 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731296508240"}]},"ts":"9223372036854775807"} 2024-11-11T03:41:48,241 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T03:41:48,241 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4216aa280844bc244aeea907e086297f, NAME => 'TestAcidGuarantees,,1731296480420.4216aa280844bc244aeea907e086297f.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T03:41:48,241 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T03:41:48,242 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731296508241"}]},"ts":"9223372036854775807"} 2024-11-11T03:41:48,243 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T03:41:48,243 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:48,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-11T03:41:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T03:41:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T03:41:48,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:48,321 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383580093_22 at /127.0.0.1:55420 [Waiting for operation #906] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1022135714_22 at /127.0.0.1:33546 [Waiting for operation #895] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383580093_22 at /127.0.0.1:33532 [Waiting for operation #870] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/3ffe7f9d089e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1022135714_22 at /127.0.0.1:33548 [Waiting for operation #884] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=295 (was 358), ProcessCount=11 (was 11), AvailableMemoryMB=11808 (was 11830) 2024-11-11T03:41:48,328 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=295, ProcessCount=11, AvailableMemoryMB=11808 2024-11-11T03:41:48,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:41:48,329 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:41:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T03:41:48,331 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T03:41:48,331 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:48,331 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-11T03:41:48,332 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T03:41:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-11T03:41:48,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742251_1427 (size=1061) 2024-11-11T03:41:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-11T03:41:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-11T03:41:48,742 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:41:48,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742252_1428 (size=53) 2024-11-11T03:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-11T03:41:49,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:41:49,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e07956485faf1a58beb177dd117c82c8, disabling compactions & flushes 2024-11-11T03:41:49,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. after waiting 0 ms 2024-11-11T03:41:49,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e07956485faf1a58beb177dd117c82c8: Waiting for close lock at 1731296509153Disabling compacts and flushes for region at 1731296509153Disabling writes for close at 1731296509153Writing region close event to WAL at 1731296509153Closed at 1731296509153 2024-11-11T03:41:49,181 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T03:41:49,181 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731296509181"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731296509181"}]},"ts":"1731296509181"} 2024-11-11T03:41:49,185 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T03:41:49,187 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T03:41:49,187 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296509187"}]},"ts":"1731296509187"} 2024-11-11T03:41:49,189 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T03:41:49,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, ASSIGN}] 2024-11-11T03:41:49,191 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, ASSIGN 2024-11-11T03:41:49,192 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, ASSIGN; state=OFFLINE, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=false 2024-11-11T03:41:49,343 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=e07956485faf1a58beb177dd117c82c8, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:49,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, ASSIGN because future has completed 2024-11-11T03:41:49,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:41:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-11T03:41:49,506 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,507 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:41:49,508 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,508 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:41:49,508 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,508 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,510 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,512 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:49,512 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e07956485faf1a58beb177dd117c82c8 columnFamilyName A 2024-11-11T03:41:49,512 DEBUG [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:49,513 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.HStore(327): Store=e07956485faf1a58beb177dd117c82c8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:49,513 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,514 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:49,515 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e07956485faf1a58beb177dd117c82c8 columnFamilyName B 2024-11-11T03:41:49,515 DEBUG [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:49,515 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.HStore(327): Store=e07956485faf1a58beb177dd117c82c8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:49,515 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,517 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:41:49,517 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e07956485faf1a58beb177dd117c82c8 columnFamilyName C 2024-11-11T03:41:49,517 DEBUG [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:41:49,518 INFO [StoreOpener-e07956485faf1a58beb177dd117c82c8-1 {}] regionserver.HStore(327): Store=e07956485faf1a58beb177dd117c82c8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:41:49,518 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,518 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,519 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,520 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,522 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,522 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,523 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:41:49,525 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,529 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:41:49,530 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened e07956485faf1a58beb177dd117c82c8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63697359, jitterRate=-0.05083538591861725}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:41:49,530 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:49,531 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for e07956485faf1a58beb177dd117c82c8: Running coprocessor pre-open hook at 1731296509508Writing region info on filesystem at 1731296509508Initializing all the Stores at 1731296509509 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296509509Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296509510 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296509510Cleaning up temporary data from old regions at 1731296509522 (+12 ms)Running coprocessor post-open hooks at 1731296509530 (+8 ms)Region opened successfully at 1731296509531 (+1 ms) 2024-11-11T03:41:49,532 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., pid=122, masterSystemTime=1731296509501 2024-11-11T03:41:49,534 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,534 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:49,534 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=e07956485faf1a58beb177dd117c82c8, regionState=OPEN, openSeqNum=2, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:49,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:41:49,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T03:41:49,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 in 190 msec 2024-11-11T03:41:49,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-11T03:41:49,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, ASSIGN in 349 msec 2024-11-11T03:41:49,543 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T03:41:49,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296509543"}]},"ts":"1731296509543"} 2024-11-11T03:41:49,545 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T03:41:49,546 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T03:41:49,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2160 sec 2024-11-11T03:41:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-11T03:41:50,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:50,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,483 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3856809e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,485 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e7b7181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T03:41:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ffe7f9d089e,45295,1731296383499 2024-11-11T03:41:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26544825 2024-11-11T03:41:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:41:50,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:41:50,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,493 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8eba1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@381b6520, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,497 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c648e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,498 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a67733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,500 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c636b62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,502 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f98b607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,504 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fd1281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f387e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bd74fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,509 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29452b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,512 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f6522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,514 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64496685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,516 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dae5644, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da04185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110431f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,521 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8322d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,523 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddd25c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299b25fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:41:50,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:41:50,527 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:41:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:41:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:41:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5ef0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:41:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:41:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:41:50,528 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:41:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30708380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:41:50,529 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,529 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,529 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:50,529 DEBUG [Thread-1933 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,529 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,530 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T03:41:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T03:41:50,532 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,532 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:50,533 DEBUG [Thread-1931 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,534 DEBUG [Thread-1935 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,536 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:50,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:50,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,538 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:41:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:41:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:41:50,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:41:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:50,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:41:50,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:41:50,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:50,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:41:50,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:50,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:41:50,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2] 2024-11-11T03:41:50,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a852c2dd5c784bcf9e13fa74ea3a04de is 50, key is test_row_0/A:col10/1731296510540/Put/seqid=0 2024-11-11T03:41:50,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742253_1429 (size=16681) 2024-11-11T03:41:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296570566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296570571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296510562, max timestamp=1731296510577Num uniques 859; 2024-11-11T03:41:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:41:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296510562, max timestamp=1731296510577Num uniques 859; 2024-11-11T03:41:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:41:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296510562, max timestamp=1731296510577Num uniques 859; 2024-11-11T03:41:50,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:41:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296570612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296570618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296570621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T03:41:50,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T03:41:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:50,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296570682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296570682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296570732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296570732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296570742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T03:41:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:50,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T03:41:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296570901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296570902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296570952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296570952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a852c2dd5c784bcf9e13fa74ea3a04de 2024-11-11T03:41:50,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296570961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:50,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/8e1a7eb9c60c4e0e99de0d0506025636 is 50, key is test_row_0/B:col10/1731296510540/Put/seqid=0 2024-11-11T03:41:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742254_1430 (size=12001) 2024-11-11T03:41:50,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/8e1a7eb9c60c4e0e99de0d0506025636 2024-11-11T03:41:50,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T03:41:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:50,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:50,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:50,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:51,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/82223b7cdf8947b6b04227fc77696299 is 50, key is test_row_0/C:col10/1731296510540/Put/seqid=0 2024-11-11T03:41:51,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742255_1431 (size=12001) 2024-11-11T03:41:51,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T03:41:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:51,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T03:41:51,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296571211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296571211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296571262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296571262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296571272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T03:41:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:51,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:51,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/82223b7cdf8947b6b04227fc77696299 2024-11-11T03:41:51,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a852c2dd5c784bcf9e13fa74ea3a04de as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a852c2dd5c784bcf9e13fa74ea3a04de 2024-11-11T03:41:51,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a852c2dd5c784bcf9e13fa74ea3a04de, entries=250, sequenceid=12, filesize=16.3 K 2024-11-11T03:41:51,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/8e1a7eb9c60c4e0e99de0d0506025636 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8e1a7eb9c60c4e0e99de0d0506025636 2024-11-11T03:41:51,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8e1a7eb9c60c4e0e99de0d0506025636, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T03:41:51,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/82223b7cdf8947b6b04227fc77696299 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/82223b7cdf8947b6b04227fc77696299 2024-11-11T03:41:51,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/82223b7cdf8947b6b04227fc77696299, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T03:41:51,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e07956485faf1a58beb177dd117c82c8 in 883ms, sequenceid=12, compaction requested=false 2024-11-11T03:41:51,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:51,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T03:41:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:51,452 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-11T03:41:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:41:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:41:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:41:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:41:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/71c92f10bd0f4328b6709b64ba8c0523 is 50, key is test_row_0/A:col10/1731296510593/Put/seqid=0 2024-11-11T03:41:51,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742256_1432 (size=12001) 2024-11-11T03:41:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T03:41:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:51,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:51,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296571762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296571767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296571782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296571782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296571791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,859 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/71c92f10bd0f4328b6709b64ba8c0523 2024-11-11T03:41:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/fec56d764c2b4251804fdab8a80af448 is 50, key is test_row_0/B:col10/1731296510593/Put/seqid=0 2024-11-11T03:41:51,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742257_1433 (size=12001) 2024-11-11T03:41:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296571882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296571882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296572102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296572102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,270 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/fec56d764c2b4251804fdab8a80af448 2024-11-11T03:41:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/31e180ea5f234bc99ce8b29892d3b784 is 50, key is test_row_0/C:col10/1731296510593/Put/seqid=0 2024-11-11T03:41:52,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742258_1434 (size=12001) 2024-11-11T03:41:52,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296572421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296572422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,680 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/31e180ea5f234bc99ce8b29892d3b784 2024-11-11T03:41:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T03:41:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/71c92f10bd0f4328b6709b64ba8c0523 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71c92f10bd0f4328b6709b64ba8c0523 2024-11-11T03:41:52,685 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71c92f10bd0f4328b6709b64ba8c0523, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T03:41:52,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/fec56d764c2b4251804fdab8a80af448 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/fec56d764c2b4251804fdab8a80af448 2024-11-11T03:41:52,688 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/fec56d764c2b4251804fdab8a80af448, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T03:41:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/31e180ea5f234bc99ce8b29892d3b784 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/31e180ea5f234bc99ce8b29892d3b784 2024-11-11T03:41:52,693 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/31e180ea5f234bc99ce8b29892d3b784, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T03:41:52,694 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for e07956485faf1a58beb177dd117c82c8 in 1243ms, sequenceid=46, compaction requested=false 2024-11-11T03:41:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T03:41:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T03:41:52,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T03:41:52,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1590 sec 2024-11-11T03:41:52,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.1680 sec 2024-11-11T03:41:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:52,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T03:41:52,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:41:52,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:52,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:41:52,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:52,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:41:52,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:52,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/7d3e1cbb64d14c80b9f2c977171d7763 is 50, key is test_row_0/A:col10/1731296512807/Put/seqid=0 2024-11-11T03:41:52,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742259_1435 (size=16681) 2024-11-11T03:41:52,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296572851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296572852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296572857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296572942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296572942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296572961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296572972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296572972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296573171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296573192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296573192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/7d3e1cbb64d14c80b9f2c977171d7763 2024-11-11T03:41:53,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/6649702cff0c4f579816e805a6f4744f is 50, key is test_row_0/B:col10/1731296512807/Put/seqid=0 2024-11-11T03:41:53,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742260_1436 (size=12001) 2024-11-11T03:41:53,422 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:41:53,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296573492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296573511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296573511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/6649702cff0c4f579816e805a6f4744f 2024-11-11T03:41:53,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/c421be37e4b747dbb852f2bd8c387ecb is 50, key is test_row_0/C:col10/1731296512807/Put/seqid=0 2024-11-11T03:41:53,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742261_1437 (size=12001) 2024-11-11T03:41:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296573952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296573952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:53,961 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in e07956485faf1a58beb177dd117c82c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:53,962 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in e07956485faf1a58beb177dd117c82c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296574011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:54,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/c421be37e4b747dbb852f2bd8c387ecb 2024-11-11T03:41:54,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/7d3e1cbb64d14c80b9f2c977171d7763 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7d3e1cbb64d14c80b9f2c977171d7763 2024-11-11T03:41:54,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296574032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:54,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296574032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:54,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7d3e1cbb64d14c80b9f2c977171d7763, entries=250, sequenceid=59, filesize=16.3 K 2024-11-11T03:41:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:54,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/6649702cff0c4f579816e805a6f4744f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/6649702cff0c4f579816e805a6f4744f 2024-11-11T03:41:54,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/6649702cff0c4f579816e805a6f4744f, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T03:41:54,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/c421be37e4b747dbb852f2bd8c387ecb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/c421be37e4b747dbb852f2bd8c387ecb 2024-11-11T03:41:54,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/c421be37e4b747dbb852f2bd8c387ecb, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T03:41:54,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e07956485faf1a58beb177dd117c82c8 in 1238ms, sequenceid=59, compaction requested=true 2024-11-11T03:41:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:54,046 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:54,046 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:54,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:54,047 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:54,047 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:54,047 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/A is initiating minor compaction (all files) 2024-11-11T03:41:54,047 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/B is initiating minor compaction (all files) 2024-11-11T03:41:54,047 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/B in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:54,047 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/A in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:54,047 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a852c2dd5c784bcf9e13fa74ea3a04de, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71c92f10bd0f4328b6709b64ba8c0523, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7d3e1cbb64d14c80b9f2c977171d7763] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=44.3 K 2024-11-11T03:41:54,047 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8e1a7eb9c60c4e0e99de0d0506025636, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/fec56d764c2b4251804fdab8a80af448, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/6649702cff0c4f579816e805a6f4744f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.2 K 2024-11-11T03:41:54,047 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e1a7eb9c60c4e0e99de0d0506025636, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731296510537 2024-11-11T03:41:54,047 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a852c2dd5c784bcf9e13fa74ea3a04de, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731296510537 2024-11-11T03:41:54,050 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting fec56d764c2b4251804fdab8a80af448, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731296510593 2024-11-11T03:41:54,050 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c92f10bd0f4328b6709b64ba8c0523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731296510593 2024-11-11T03:41:54,050 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3e1cbb64d14c80b9f2c977171d7763, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731296511745 2024-11-11T03:41:54,050 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6649702cff0c4f579816e805a6f4744f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731296511745 2024-11-11T03:41:54,057 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#A#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:54,057 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/73cbc213c3a143f4ac226f871edbcc03 is 50, key is test_row_0/A:col10/1731296512807/Put/seqid=0 2024-11-11T03:41:54,069 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:54,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742262_1438 (size=12104) 2024-11-11T03:41:54,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/300ad79011854deda2f4edba3ed79dfd is 50, key is test_row_0/B:col10/1731296512807/Put/seqid=0 2024-11-11T03:41:54,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742263_1439 (size=12104) 2024-11-11T03:41:54,081 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/300ad79011854deda2f4edba3ed79dfd as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/300ad79011854deda2f4edba3ed79dfd 2024-11-11T03:41:54,085 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/B of e07956485faf1a58beb177dd117c82c8 into 300ad79011854deda2f4edba3ed79dfd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:54,085 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:54,085 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/B, priority=13, startTime=1731296514046; duration=0sec 2024-11-11T03:41:54,085 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:54,085 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:41:54,085 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:54,086 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:54,086 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/C is initiating minor compaction (all files) 2024-11-11T03:41:54,086 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/C in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:54,086 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/82223b7cdf8947b6b04227fc77696299, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/31e180ea5f234bc99ce8b29892d3b784, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/c421be37e4b747dbb852f2bd8c387ecb] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.2 K 2024-11-11T03:41:54,087 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 82223b7cdf8947b6b04227fc77696299, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731296510537 2024-11-11T03:41:54,087 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e180ea5f234bc99ce8b29892d3b784, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731296510593 2024-11-11T03:41:54,087 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting c421be37e4b747dbb852f2bd8c387ecb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731296511745 2024-11-11T03:41:54,096 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:54,096 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/81b0b1e036de40e09fd958b7c8d0fa8d is 50, key is test_row_0/C:col10/1731296512807/Put/seqid=0 2024-11-11T03:41:54,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742264_1440 (size=12104) 2024-11-11T03:41:54,473 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/73cbc213c3a143f4ac226f871edbcc03 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/73cbc213c3a143f4ac226f871edbcc03 2024-11-11T03:41:54,476 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/A of e07956485faf1a58beb177dd117c82c8 into 73cbc213c3a143f4ac226f871edbcc03(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:54,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:54,476 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/A, priority=13, startTime=1731296514046; duration=0sec 2024-11-11T03:41:54,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:54,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:41:54,503 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/81b0b1e036de40e09fd958b7c8d0fa8d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/81b0b1e036de40e09fd958b7c8d0fa8d 2024-11-11T03:41:54,507 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/C of e07956485faf1a58beb177dd117c82c8 into 81b0b1e036de40e09fd958b7c8d0fa8d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:54,507 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:54,507 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/C, priority=13, startTime=1731296514046; duration=0sec 2024-11-11T03:41:54,507 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:54,507 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:41:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T03:41:54,702 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:54,702 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T03:41:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T03:41:54,704 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:54,704 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:54,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T03:41:54,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T03:41:54,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:54,856 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T03:41:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:41:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:41:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:41:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/ac0f5ac1fbe74190ae09fc20346da38a is 50, key is test_row_0/A:col10/1731296512856/Put/seqid=0 2024-11-11T03:41:54,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742265_1441 (size=12001) 2024-11-11T03:41:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T03:41:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:55,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:55,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296575061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296575063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296575070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296575182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296575182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296575182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,264 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/ac0f5ac1fbe74190ae09fc20346da38a 2024-11-11T03:41:55,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/1660e08536ae4dc884804c2d1649c459 is 50, key is test_row_0/B:col10/1731296512856/Put/seqid=0 2024-11-11T03:41:55,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742266_1442 (size=12001) 2024-11-11T03:41:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T03:41:55,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296575392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296575392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296575392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,674 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/1660e08536ae4dc884804c2d1649c459 2024-11-11T03:41:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/93c2d21563a8480c9cf5948f63502ae4 is 50, key is test_row_0/C:col10/1731296512856/Put/seqid=0 2024-11-11T03:41:55,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742267_1443 (size=12001) 2024-11-11T03:41:55,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296575711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296575711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296575711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T03:41:55,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296575972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,979 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in e07956485faf1a58beb177dd117c82c8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:55,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296575982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:55,989 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in e07956485faf1a58beb177dd117c82c8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:41:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,083 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/93c2d21563a8480c9cf5948f63502ae4 2024-11-11T03:41:56,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/ac0f5ac1fbe74190ae09fc20346da38a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/ac0f5ac1fbe74190ae09fc20346da38a 2024-11-11T03:41:56,089 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/ac0f5ac1fbe74190ae09fc20346da38a, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T03:41:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/1660e08536ae4dc884804c2d1649c459 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1660e08536ae4dc884804c2d1649c459 2024-11-11T03:41:56,092 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1660e08536ae4dc884804c2d1649c459, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T03:41:56,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/93c2d21563a8480c9cf5948f63502ae4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/93c2d21563a8480c9cf5948f63502ae4 2024-11-11T03:41:56,096 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/93c2d21563a8480c9cf5948f63502ae4, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T03:41:56,097 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e07956485faf1a58beb177dd117c82c8 in 1241ms, sequenceid=81, compaction requested=false 2024-11-11T03:41:56,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:56,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:56,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T03:41:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T03:41:56,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T03:41:56,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3940 sec 2024-11-11T03:41:56,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3980 sec 2024-11-11T03:41:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:56,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T03:41:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:41:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:41:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:41:56,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:56,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/98022d3f2a6a43929ccc2a2dfe861635 is 50, key is test_row_0/A:col10/1731296515043/Put/seqid=0 2024-11-11T03:41:56,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742268_1444 (size=14341) 2024-11-11T03:41:56,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296576270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296576271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296576275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296576381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296576392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296576392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296576601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296576612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296576612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/98022d3f2a6a43929ccc2a2dfe861635 2024-11-11T03:41:56,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/3390279e9ea546e7a34e779787e0275a is 50, key is test_row_0/B:col10/1731296515043/Put/seqid=0 2024-11-11T03:41:56,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742269_1445 (size=12001) 2024-11-11T03:41:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T03:41:56,852 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:56,852 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T03:41:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T03:41:56,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:56,855 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:56,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:56,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296576922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296576922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296576922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T03:41:57,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T03:41:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/3390279e9ea546e7a34e779787e0275a 2024-11-11T03:41:57,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/7afe6fc59f874e2993851f1ee3442e3f is 50, key is test_row_0/C:col10/1731296515043/Put/seqid=0 2024-11-11T03:41:57,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742270_1446 (size=12001) 2024-11-11T03:41:57,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T03:41:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T03:41:57,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T03:41:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:57,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296577432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296577432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:57,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296577441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:57,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/7afe6fc59f874e2993851f1ee3442e3f 2024-11-11T03:41:57,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/98022d3f2a6a43929ccc2a2dfe861635 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/98022d3f2a6a43929ccc2a2dfe861635 2024-11-11T03:41:57,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/98022d3f2a6a43929ccc2a2dfe861635, entries=200, sequenceid=96, filesize=14.0 K 2024-11-11T03:41:57,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/3390279e9ea546e7a34e779787e0275a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/3390279e9ea546e7a34e779787e0275a 2024-11-11T03:41:57,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/3390279e9ea546e7a34e779787e0275a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T03:41:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/7afe6fc59f874e2993851f1ee3442e3f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/7afe6fc59f874e2993851f1ee3442e3f 2024-11-11T03:41:57,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/7afe6fc59f874e2993851f1ee3442e3f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T03:41:57,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e07956485faf1a58beb177dd117c82c8 in 1236ms, sequenceid=96, compaction requested=true 2024-11-11T03:41:57,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:41:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:57,464 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:57,464 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:41:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:41:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:57,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T03:41:57,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,465 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:41:57,465 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:57,465 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:57,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:41:57,465 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/A is initiating minor compaction (all files) 2024-11-11T03:41:57,465 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/B is initiating minor compaction (all files) 2024-11-11T03:41:57,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:57,465 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/A in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:41:57,465 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/B in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:57,465 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/73cbc213c3a143f4ac226f871edbcc03, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/ac0f5ac1fbe74190ae09fc20346da38a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/98022d3f2a6a43929ccc2a2dfe861635] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=37.5 K 2024-11-11T03:41:57,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:41:57,465 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/300ad79011854deda2f4edba3ed79dfd, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1660e08536ae4dc884804c2d1649c459, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/3390279e9ea546e7a34e779787e0275a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.3 K 2024-11-11T03:41:57,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:57,465 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 300ad79011854deda2f4edba3ed79dfd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731296511745 2024-11-11T03:41:57,465 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73cbc213c3a143f4ac226f871edbcc03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731296511745 2024-11-11T03:41:57,466 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0f5ac1fbe74190ae09fc20346da38a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296512836 2024-11-11T03:41:57,466 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 1660e08536ae4dc884804c2d1649c459, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296512836 2024-11-11T03:41:57,466 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3390279e9ea546e7a34e779787e0275a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731296515043 2024-11-11T03:41:57,466 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98022d3f2a6a43929ccc2a2dfe861635, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731296515043 2024-11-11T03:41:57,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/d2a2a623bd00453c9e4265c9aa90b032 is 50, key is test_row_0/A:col10/1731296516273/Put/seqid=0 2024-11-11T03:41:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T03:41:57,484 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:57,484 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#A#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:57,485 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/b0ac2aaf7edd4af0b0b86f39118c5e40 is 50, key is test_row_0/B:col10/1731296515043/Put/seqid=0 2024-11-11T03:41:57,485 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/410a1f9f6a094d988ecda51847b066bf is 50, key is test_row_0/A:col10/1731296515043/Put/seqid=0 2024-11-11T03:41:57,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742271_1447 (size=12001) 2024-11-11T03:41:57,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742272_1448 (size=12207) 2024-11-11T03:41:57,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742273_1449 (size=12207) 2024-11-11T03:41:57,495 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/410a1f9f6a094d988ecda51847b066bf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/410a1f9f6a094d988ecda51847b066bf 2024-11-11T03:41:57,499 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/A of e07956485faf1a58beb177dd117c82c8 into 410a1f9f6a094d988ecda51847b066bf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:57,499 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:57,499 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/A, priority=13, startTime=1731296517463; duration=0sec 2024-11-11T03:41:57,499 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:41:57,499 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:41:57,499 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:41:57,500 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:41:57,500 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/C is initiating minor compaction (all files) 2024-11-11T03:41:57,500 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/C in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:57,500 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/81b0b1e036de40e09fd958b7c8d0fa8d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/93c2d21563a8480c9cf5948f63502ae4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/7afe6fc59f874e2993851f1ee3442e3f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.3 K 2024-11-11T03:41:57,501 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b0b1e036de40e09fd958b7c8d0fa8d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731296511745 2024-11-11T03:41:57,501 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c2d21563a8480c9cf5948f63502ae4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296512836 2024-11-11T03:41:57,501 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7afe6fc59f874e2993851f1ee3442e3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731296515043 2024-11-11T03:41:57,506 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:41:57,506 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/08ef6277e6a14d21b000ac155aeeff9e is 50, key is test_row_0/C:col10/1731296515043/Put/seqid=0 2024-11-11T03:41:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742274_1450 (size=12207) 2024-11-11T03:41:57,885 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/d2a2a623bd00453c9e4265c9aa90b032 2024-11-11T03:41:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/53f8338a3bae4a2eb57a571861d65f0e is 50, key is test_row_0/B:col10/1731296516273/Put/seqid=0 2024-11-11T03:41:57,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742275_1451 (size=12001) 2024-11-11T03:41:57,897 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/b0ac2aaf7edd4af0b0b86f39118c5e40 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b0ac2aaf7edd4af0b0b86f39118c5e40 2024-11-11T03:41:57,901 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/B of e07956485faf1a58beb177dd117c82c8 into b0ac2aaf7edd4af0b0b86f39118c5e40(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:57,901 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:57,901 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/B, priority=13, startTime=1731296517464; duration=0sec 2024-11-11T03:41:57,901 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:57,901 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:41:57,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/08ef6277e6a14d21b000ac155aeeff9e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/08ef6277e6a14d21b000ac155aeeff9e 2024-11-11T03:41:57,916 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/C of e07956485faf1a58beb177dd117c82c8 into 08ef6277e6a14d21b000ac155aeeff9e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:41:57,916 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:57,916 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/C, priority=13, startTime=1731296517464; duration=0sec 2024-11-11T03:41:57,916 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:41:57,916 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:41:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T03:41:58,295 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/53f8338a3bae4a2eb57a571861d65f0e 2024-11-11T03:41:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/04b61c393151462387601e930870cc1d is 50, key is test_row_0/C:col10/1731296516273/Put/seqid=0 2024-11-11T03:41:58,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742276_1452 (size=12001) 2024-11-11T03:41:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:58,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296578485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296578490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296578491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296578601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296578601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296578612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,704 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/04b61c393151462387601e930870cc1d 2024-11-11T03:41:58,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/d2a2a623bd00453c9e4265c9aa90b032 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/d2a2a623bd00453c9e4265c9aa90b032 2024-11-11T03:41:58,710 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/d2a2a623bd00453c9e4265c9aa90b032, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T03:41:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/53f8338a3bae4a2eb57a571861d65f0e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/53f8338a3bae4a2eb57a571861d65f0e 2024-11-11T03:41:58,714 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/53f8338a3bae4a2eb57a571861d65f0e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T03:41:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/04b61c393151462387601e930870cc1d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/04b61c393151462387601e930870cc1d 2024-11-11T03:41:58,717 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/04b61c393151462387601e930870cc1d, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T03:41:58,718 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e07956485faf1a58beb177dd117c82c8 in 1253ms, sequenceid=113, compaction requested=false 2024-11-11T03:41:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:41:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T03:41:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T03:41:58,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T03:41:58,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-11-11T03:41:58,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8690 sec 2024-11-11T03:41:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:41:58,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T03:41:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:41:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:41:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:41:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:41:58,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a1195a592de042f3939fdfce2a4d76fe is 50, key is test_row_0/A:col10/1731296518827/Put/seqid=0 2024-11-11T03:41:58,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742277_1453 (size=16881) 2024-11-11T03:41:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296578857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296578859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296578860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296578982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296578982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:58,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296578982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T03:41:59,002 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:41:59,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:41:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T03:41:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T03:41:59,004 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:41:59,004 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:41:59,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:41:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T03:41:59,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T03:41:59,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296579192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296579192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296579192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:59,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a1195a592de042f3939fdfce2a4d76fe 2024-11-11T03:41:59,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f27c53ffc13c43d1b92d90ecc935a227 is 50, key is test_row_0/B:col10/1731296518827/Put/seqid=0 2024-11-11T03:41:59,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742278_1454 (size=12101) 2024-11-11T03:41:59,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T03:41:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T03:41:59,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T03:41:59,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:59,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296579512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:59,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296579512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:59,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296579512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:41:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:41:59,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T03:41:59,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:59,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T03:41:59,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f27c53ffc13c43d1b92d90ecc935a227 2024-11-11T03:41:59,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/ea302e100b014838aa927da775388e96 is 50, key is test_row_0/C:col10/1731296518827/Put/seqid=0 2024-11-11T03:41:59,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742279_1455 (size=12101) 2024-11-11T03:41:59,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T03:41:59,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:59,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T03:41:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:41:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:41:59,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:41:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:00,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296580012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:00,017 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in e07956485faf1a58beb177dd117c82c8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:00,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296580022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:00,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296580022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:00,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296580022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:00,029 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in e07956485faf1a58beb177dd117c82c8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:00,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296580022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:00,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:00,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:00,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/ea302e100b014838aa927da775388e96 2024-11-11T03:42:00,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a1195a592de042f3939fdfce2a4d76fe as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a1195a592de042f3939fdfce2a4d76fe 2024-11-11T03:42:00,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a1195a592de042f3939fdfce2a4d76fe, entries=250, sequenceid=133, filesize=16.5 K 2024-11-11T03:42:00,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f27c53ffc13c43d1b92d90ecc935a227 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f27c53ffc13c43d1b92d90ecc935a227 2024-11-11T03:42:00,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f27c53ffc13c43d1b92d90ecc935a227, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T03:42:00,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/ea302e100b014838aa927da775388e96 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ea302e100b014838aa927da775388e96 2024-11-11T03:42:00,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ea302e100b014838aa927da775388e96, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T03:42:00,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e07956485faf1a58beb177dd117c82c8 in 1239ms, sequenceid=133, compaction requested=true 2024-11-11T03:42:00,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:00,068 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:00,068 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:00,068 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/A is initiating minor compaction (all files) 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:00,069 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/A in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/B is initiating minor compaction (all files) 2024-11-11T03:42:00,069 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/B in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:00,069 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/410a1f9f6a094d988ecda51847b066bf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/d2a2a623bd00453c9e4265c9aa90b032, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a1195a592de042f3939fdfce2a4d76fe] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=40.1 K 2024-11-11T03:42:00,069 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b0ac2aaf7edd4af0b0b86f39118c5e40, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/53f8338a3bae4a2eb57a571861d65f0e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f27c53ffc13c43d1b92d90ecc935a227] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.5 K 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 410a1f9f6a094d988ecda51847b066bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731296515043 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ac2aaf7edd4af0b0b86f39118c5e40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731296515043 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a2a623bd00453c9e4265c9aa90b032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731296516268 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 53f8338a3bae4a2eb57a571861d65f0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731296516268 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1195a592de042f3939fdfce2a4d76fe, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731296518483 2024-11-11T03:42:00,069 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f27c53ffc13c43d1b92d90ecc935a227, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731296518483 2024-11-11T03:42:00,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T03:42:00,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:00,071 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:42:00,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/14ba7b1a53ca4d01b045b137d51c1098 is 50, key is test_row_0/A:col10/1731296518857/Put/seqid=0 2024-11-11T03:42:00,079 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:00,079 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/0a9f7af96af34161a0ffbfbc9dce48d6 is 50, key is test_row_0/A:col10/1731296518827/Put/seqid=0 2024-11-11T03:42:00,085 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:00,085 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ba50621893de4f86b7c14c695fd316c1 is 50, key is test_row_0/B:col10/1731296518827/Put/seqid=0 2024-11-11T03:42:00,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742281_1457 (size=12409) 2024-11-11T03:42:00,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742282_1458 (size=12409) 2024-11-11T03:42:00,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742280_1456 (size=12151) 2024-11-11T03:42:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T03:42:00,496 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/0a9f7af96af34161a0ffbfbc9dce48d6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/0a9f7af96af34161a0ffbfbc9dce48d6 2024-11-11T03:42:00,500 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/14ba7b1a53ca4d01b045b137d51c1098 2024-11-11T03:42:00,501 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/A of e07956485faf1a58beb177dd117c82c8 into 0a9f7af96af34161a0ffbfbc9dce48d6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:00,501 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:00,502 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/A, priority=13, startTime=1731296520068; duration=0sec 2024-11-11T03:42:00,502 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:00,502 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:42:00,502 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:00,502 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ba50621893de4f86b7c14c695fd316c1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba50621893de4f86b7c14c695fd316c1 2024-11-11T03:42:00,502 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:00,503 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/C is initiating minor compaction (all files) 2024-11-11T03:42:00,503 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/C in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:00,503 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/08ef6277e6a14d21b000ac155aeeff9e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/04b61c393151462387601e930870cc1d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ea302e100b014838aa927da775388e96] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.5 K 2024-11-11T03:42:00,503 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ef6277e6a14d21b000ac155aeeff9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731296515043 2024-11-11T03:42:00,503 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b61c393151462387601e930870cc1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731296516268 2024-11-11T03:42:00,503 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea302e100b014838aa927da775388e96, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731296518483 2024-11-11T03:42:00,506 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/B of e07956485faf1a58beb177dd117c82c8 into ba50621893de4f86b7c14c695fd316c1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:00,506 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:00,506 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/B, priority=13, startTime=1731296520068; duration=0sec 2024-11-11T03:42:00,506 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:00,506 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:42:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ce10d09f6bbc48c8afc3c81458eeff7c is 50, key is test_row_0/B:col10/1731296518857/Put/seqid=0 2024-11-11T03:42:00,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742283_1459 (size=12151) 2024-11-11T03:42:00,510 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:00,510 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ce10d09f6bbc48c8afc3c81458eeff7c 2024-11-11T03:42:00,510 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/ba2839bc3edb466f92422e3653da2da1 is 50, key is test_row_0/C:col10/1731296518827/Put/seqid=0 2024-11-11T03:42:00,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742284_1460 (size=12409) 2024-11-11T03:42:00,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/6c0f33f805ba44a5805fa8b4650fe08e is 50, key is test_row_0/C:col10/1731296518857/Put/seqid=0 2024-11-11T03:42:00,521 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/ba2839bc3edb466f92422e3653da2da1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ba2839bc3edb466f92422e3653da2da1 2024-11-11T03:42:00,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742285_1461 (size=12151) 2024-11-11T03:42:00,525 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/C of e07956485faf1a58beb177dd117c82c8 into ba2839bc3edb466f92422e3653da2da1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:00,525 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:00,525 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/C, priority=13, startTime=1731296520068; duration=0sec 2024-11-11T03:42:00,525 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:00,525 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:42:00,922 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/6c0f33f805ba44a5805fa8b4650fe08e 2024-11-11T03:42:00,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/14ba7b1a53ca4d01b045b137d51c1098 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/14ba7b1a53ca4d01b045b137d51c1098 2024-11-11T03:42:00,928 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/14ba7b1a53ca4d01b045b137d51c1098, entries=150, sequenceid=148, filesize=11.9 K 2024-11-11T03:42:00,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ce10d09f6bbc48c8afc3c81458eeff7c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ce10d09f6bbc48c8afc3c81458eeff7c 2024-11-11T03:42:00,932 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ce10d09f6bbc48c8afc3c81458eeff7c, entries=150, sequenceid=148, filesize=11.9 K 2024-11-11T03:42:00,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/6c0f33f805ba44a5805fa8b4650fe08e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/6c0f33f805ba44a5805fa8b4650fe08e 2024-11-11T03:42:00,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/6c0f33f805ba44a5805fa8b4650fe08e, entries=150, sequenceid=148, filesize=11.9 K 2024-11-11T03:42:00,937 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for e07956485faf1a58beb177dd117c82c8 in 866ms, sequenceid=148, compaction requested=false 2024-11-11T03:42:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T03:42:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T03:42:00,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T03:42:00,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9340 sec 2024-11-11T03:42:00,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9380 sec 2024-11-11T03:42:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:01,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:42:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:01,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/bebe273dec9848bf9640b8cf415a5c44 is 50, key is test_row_0/A:col10/1731296521045/Put/seqid=0 2024-11-11T03:42:01,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742286_1462 (size=14541) 2024-11-11T03:42:01,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296521088, max timestamp=1731296521113Num uniques 859; 2024-11-11T03:42:01,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296521088, max timestamp=1731296521113Num uniques 859; 2024-11-11T03:42:01,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296521088, max timestamp=1731296521113Num uniques 859; 2024-11-11T03:42:01,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:42:01,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:42:01,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:42:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T03:42:01,152 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:01,152 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-11T03:42:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T03:42:01,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:01,155 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:01,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:01,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296581161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296581162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296581177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T03:42:01,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296581282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296581282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296581292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T03:42:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T03:42:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/bebe273dec9848bf9640b8cf415a5c44 2024-11-11T03:42:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T03:42:01,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/5fd5fddc15b44e0985837f6e7359f2d4 is 50, key is test_row_0/B:col10/1731296521045/Put/seqid=0 2024-11-11T03:42:01,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742287_1463 (size=12151) 2024-11-11T03:42:01,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296581501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296581501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296581502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T03:42:01,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:01,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T03:42:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T03:42:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296581822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296581822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296581822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:01,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/5fd5fddc15b44e0985837f6e7359f2d4 2024-11-11T03:42:01,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/94ab2d948333445398893ac45f37a74d is 50, key is test_row_0/C:col10/1731296521045/Put/seqid=0 2024-11-11T03:42:01,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742288_1464 (size=12151) 2024-11-11T03:42:01,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/94ab2d948333445398893ac45f37a74d 2024-11-11T03:42:01,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/bebe273dec9848bf9640b8cf415a5c44 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/bebe273dec9848bf9640b8cf415a5c44 2024-11-11T03:42:01,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/bebe273dec9848bf9640b8cf415a5c44, entries=200, sequenceid=161, filesize=14.2 K 2024-11-11T03:42:01,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/5fd5fddc15b44e0985837f6e7359f2d4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/5fd5fddc15b44e0985837f6e7359f2d4 2024-11-11T03:42:01,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/5fd5fddc15b44e0985837f6e7359f2d4, entries=150, sequenceid=161, filesize=11.9 K 2024-11-11T03:42:01,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/94ab2d948333445398893ac45f37a74d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/94ab2d948333445398893ac45f37a74d 2024-11-11T03:42:01,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/94ab2d948333445398893ac45f37a74d, entries=150, sequenceid=161, filesize=11.9 K 2024-11-11T03:42:01,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e07956485faf1a58beb177dd117c82c8 in 853ms, sequenceid=161, compaction requested=true 2024-11-11T03:42:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:01,904 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:01,904 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:01,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:01,904 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:01,904 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/B is initiating minor compaction (all files) 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/A is initiating minor compaction (all files) 2024-11-11T03:42:01,905 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/A in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,905 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/B in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,905 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba50621893de4f86b7c14c695fd316c1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ce10d09f6bbc48c8afc3c81458eeff7c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/5fd5fddc15b44e0985837f6e7359f2d4] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.9 K 2024-11-11T03:42:01,905 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/0a9f7af96af34161a0ffbfbc9dce48d6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/14ba7b1a53ca4d01b045b137d51c1098, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/bebe273dec9848bf9640b8cf415a5c44] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=38.2 K 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ba50621893de4f86b7c14c695fd316c1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731296518483 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9f7af96af34161a0ffbfbc9dce48d6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731296518483 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ce10d09f6bbc48c8afc3c81458eeff7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731296518855 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ba7b1a53ca4d01b045b137d51c1098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731296518855 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd5fddc15b44e0985837f6e7359f2d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731296521045 2024-11-11T03:42:01,905 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting bebe273dec9848bf9640b8cf415a5c44, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731296521045 2024-11-11T03:42:01,911 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:01,911 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/60bee191f02b4a389a222c11e9cfe6c2 is 50, key is test_row_0/B:col10/1731296521045/Put/seqid=0 2024-11-11T03:42:01,911 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:01,911 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/44c349ca263b41008dff7e79b82a9f46 is 50, key is test_row_0/A:col10/1731296521045/Put/seqid=0 2024-11-11T03:42:01,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742290_1466 (size=12561) 2024-11-11T03:42:01,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742289_1465 (size=12561) 2024-11-11T03:42:01,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T03:42:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:01,917 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T03:42:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:01,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/2d3dfe24e1c04ae68b881b8cec7f786f is 50, key is test_row_0/A:col10/1731296521161/Put/seqid=0 2024-11-11T03:42:01,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742291_1467 (size=12151) 2024-11-11T03:42:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T03:42:02,317 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/60bee191f02b4a389a222c11e9cfe6c2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/60bee191f02b4a389a222c11e9cfe6c2 2024-11-11T03:42:02,317 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/44c349ca263b41008dff7e79b82a9f46 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/44c349ca263b41008dff7e79b82a9f46 2024-11-11T03:42:02,320 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/B of e07956485faf1a58beb177dd117c82c8 into 60bee191f02b4a389a222c11e9cfe6c2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:02,320 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/A of e07956485faf1a58beb177dd117c82c8 into 44c349ca263b41008dff7e79b82a9f46(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:02,320 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:02,320 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:02,320 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/A, priority=13, startTime=1731296521904; duration=0sec 2024-11-11T03:42:02,320 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/B, priority=13, startTime=1731296521904; duration=0sec 2024-11-11T03:42:02,320 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:02,320 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:42:02,320 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:02,320 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:42:02,321 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:02,321 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:02,321 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/C is initiating minor compaction (all files) 2024-11-11T03:42:02,321 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/C in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:02,321 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ba2839bc3edb466f92422e3653da2da1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/6c0f33f805ba44a5805fa8b4650fe08e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/94ab2d948333445398893ac45f37a74d] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=35.9 K 2024-11-11T03:42:02,322 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba2839bc3edb466f92422e3653da2da1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731296518483 2024-11-11T03:42:02,322 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c0f33f805ba44a5805fa8b4650fe08e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731296518855 2024-11-11T03:42:02,322 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ab2d948333445398893ac45f37a74d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731296521045 2024-11-11T03:42:02,326 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/2d3dfe24e1c04ae68b881b8cec7f786f 2024-11-11T03:42:02,328 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:02,329 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/39c37f41356240e0be5f22966bea5ee8 is 50, key is test_row_0/C:col10/1731296521045/Put/seqid=0 2024-11-11T03:42:02,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/785d4cb41fa24f489d644ea6ef79c420 is 50, key is test_row_0/B:col10/1731296521161/Put/seqid=0 2024-11-11T03:42:02,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742292_1468 (size=12561) 2024-11-11T03:42:02,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742293_1469 (size=12151) 2024-11-11T03:42:02,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/785d4cb41fa24f489d644ea6ef79c420 2024-11-11T03:42:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:02,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:02,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/0257d6b6f1bf49b7a7eef7431836c7d2 is 50, key is test_row_0/C:col10/1731296521161/Put/seqid=0 2024-11-11T03:42:02,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742294_1470 (size=12151) 2024-11-11T03:42:02,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296582366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296582366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296582369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296582481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296582481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296582482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296582701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296582701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296582701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:02,741 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/39c37f41356240e0be5f22966bea5ee8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/39c37f41356240e0be5f22966bea5ee8 2024-11-11T03:42:02,743 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/C of e07956485faf1a58beb177dd117c82c8 into 39c37f41356240e0be5f22966bea5ee8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:02,744 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:02,744 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/C, priority=13, startTime=1731296521904; duration=0sec 2024-11-11T03:42:02,744 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:02,744 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:42:02,753 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/0257d6b6f1bf49b7a7eef7431836c7d2 2024-11-11T03:42:02,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/2d3dfe24e1c04ae68b881b8cec7f786f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2d3dfe24e1c04ae68b881b8cec7f786f 2024-11-11T03:42:02,759 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2d3dfe24e1c04ae68b881b8cec7f786f, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T03:42:02,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/785d4cb41fa24f489d644ea6ef79c420 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/785d4cb41fa24f489d644ea6ef79c420 2024-11-11T03:42:02,762 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/785d4cb41fa24f489d644ea6ef79c420, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T03:42:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/0257d6b6f1bf49b7a7eef7431836c7d2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/0257d6b6f1bf49b7a7eef7431836c7d2 2024-11-11T03:42:02,765 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/0257d6b6f1bf49b7a7eef7431836c7d2, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T03:42:02,766 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for e07956485faf1a58beb177dd117c82c8 in 849ms, sequenceid=196, compaction requested=false 2024-11-11T03:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-11T03:42:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-11T03:42:02,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T03:42:02,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-11-11T03:42:02,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6170 sec 2024-11-11T03:42:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:03,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T03:42:03,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:03,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:03,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:03,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:03,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:03,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:03,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/be840fa0035843b989ede00cc0c131e7 is 50, key is test_row_0/A:col10/1731296522367/Put/seqid=0 2024-11-11T03:42:03,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742295_1471 (size=14541) 2024-11-11T03:42:03,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296583062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296523049, max timestamp=1731296523065Num uniques 859; 2024-11-11T03:42:03,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:42:03,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296523049, max timestamp=1731296523065Num uniques 859; 2024-11-11T03:42:03,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:42:03,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731296523049, max timestamp=1731296523065Num uniques 859; 2024-11-11T03:42:03,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T03:42:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296583124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296583131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296583171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296583242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296583242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T03:42:03,302 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:03,302 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-11T03:42:03,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T03:42:03,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:03,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:03,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296583391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T03:42:03,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/be840fa0035843b989ede00cc0c131e7 2024-11-11T03:42:03,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f0b2f80050264a8a8ce4a6e434b12090 is 50, key is test_row_0/B:col10/1731296522367/Put/seqid=0 2024-11-11T03:42:03,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742296_1472 (size=12151) 2024-11-11T03:42:03,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296583462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296583462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T03:42:03,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296583701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296583772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296583772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:03,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f0b2f80050264a8a8ce4a6e434b12090 2024-11-11T03:42:03,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/79620ddb7924483680a90af17557566f is 50, key is test_row_0/C:col10/1731296522367/Put/seqid=0 2024-11-11T03:42:03,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742297_1473 (size=12151) 2024-11-11T03:42:03,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:03,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:03,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:03,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T03:42:04,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296584211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/79620ddb7924483680a90af17557566f 2024-11-11T03:42:04,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/be840fa0035843b989ede00cc0c131e7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/be840fa0035843b989ede00cc0c131e7 2024-11-11T03:42:04,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/be840fa0035843b989ede00cc0c131e7, entries=200, sequenceid=210, filesize=14.2 K 2024-11-11T03:42:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f0b2f80050264a8a8ce4a6e434b12090 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f0b2f80050264a8a8ce4a6e434b12090 2024-11-11T03:42:04,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f0b2f80050264a8a8ce4a6e434b12090, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T03:42:04,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/79620ddb7924483680a90af17557566f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/79620ddb7924483680a90af17557566f 2024-11-11T03:42:04,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/79620ddb7924483680a90af17557566f, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T03:42:04,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e07956485faf1a58beb177dd117c82c8 in 1237ms, sequenceid=210, compaction requested=true 2024-11-11T03:42:04,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:04,252 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:04,252 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:04,252 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:04,252 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:04,252 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/B is initiating minor compaction (all files) 2024-11-11T03:42:04,252 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/A is initiating minor compaction (all files) 2024-11-11T03:42:04,252 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/B in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,252 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/A in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,253 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/60bee191f02b4a389a222c11e9cfe6c2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/785d4cb41fa24f489d644ea6ef79c420, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f0b2f80050264a8a8ce4a6e434b12090] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=36.0 K 2024-11-11T03:42:04,253 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/44c349ca263b41008dff7e79b82a9f46, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2d3dfe24e1c04ae68b881b8cec7f786f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/be840fa0035843b989ede00cc0c131e7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=38.3 K 2024-11-11T03:42:04,253 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44c349ca263b41008dff7e79b82a9f46, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731296521045 2024-11-11T03:42:04,253 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 60bee191f02b4a389a222c11e9cfe6c2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731296521045 2024-11-11T03:42:04,253 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 785d4cb41fa24f489d644ea6ef79c420, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731296521156 2024-11-11T03:42:04,253 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d3dfe24e1c04ae68b881b8cec7f786f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731296521156 2024-11-11T03:42:04,253 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f0b2f80050264a8a8ce4a6e434b12090, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731296522365 2024-11-11T03:42:04,253 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting be840fa0035843b989ede00cc0c131e7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731296522348 2024-11-11T03:42:04,260 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#B#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:04,260 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:04,260 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/46747eedc4064a7e94faf3c95905911d is 50, key is test_row_0/B:col10/1731296522367/Put/seqid=0 2024-11-11T03:42:04,260 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/753e884281e34f99a1a80a1fc543cdb4 is 50, key is test_row_0/A:col10/1731296522367/Put/seqid=0 2024-11-11T03:42:04,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742299_1475 (size=12663) 2024-11-11T03:42:04,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742298_1474 (size=12663) 2024-11-11T03:42:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:04,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T03:42:04,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:04,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:04,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:04,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:04,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:04,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:04,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a36a317442f6409e93a844c52ef666b6 is 50, key is test_row_0/A:col10/1731296524298/Put/seqid=0 2024-11-11T03:42:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742300_1476 (size=14541) 2024-11-11T03:42:04,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296584329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296584336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T03:42:04,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296584442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296584452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,667 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/46747eedc4064a7e94faf3c95905911d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/46747eedc4064a7e94faf3c95905911d 2024-11-11T03:42:04,667 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/753e884281e34f99a1a80a1fc543cdb4 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/753e884281e34f99a1a80a1fc543cdb4 2024-11-11T03:42:04,673 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/B of e07956485faf1a58beb177dd117c82c8 into 46747eedc4064a7e94faf3c95905911d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:04,673 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:04,673 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/B, priority=13, startTime=1731296524252; duration=0sec 2024-11-11T03:42:04,673 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:04,673 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:42:04,673 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:04,674 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/A of e07956485faf1a58beb177dd117c82c8 into 753e884281e34f99a1a80a1fc543cdb4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:04,674 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:04,674 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/A, priority=13, startTime=1731296524252; duration=0sec 2024-11-11T03:42:04,674 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:04,674 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:42:04,674 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:04,674 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/C is initiating minor compaction (all files) 2024-11-11T03:42:04,674 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/C in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,674 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/39c37f41356240e0be5f22966bea5ee8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/0257d6b6f1bf49b7a7eef7431836c7d2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/79620ddb7924483680a90af17557566f] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=36.0 K 2024-11-11T03:42:04,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296584661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,675 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c37f41356240e0be5f22966bea5ee8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731296521045 2024-11-11T03:42:04,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296584672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,676 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0257d6b6f1bf49b7a7eef7431836c7d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731296521156 2024-11-11T03:42:04,676 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 79620ddb7924483680a90af17557566f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731296522365 2024-11-11T03:42:04,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:04,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:04,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,682 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:04,682 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/4702dbd912a241e6bd64c03e80a0b1bb is 50, key is test_row_0/C:col10/1731296522367/Put/seqid=0 2024-11-11T03:42:04,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742301_1477 (size=12663) 2024-11-11T03:42:04,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a36a317442f6409e93a844c52ef666b6 2024-11-11T03:42:04,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/248a1bc246c84c649f968f550ff5b67e is 50, key is test_row_0/B:col10/1731296524298/Put/seqid=0 2024-11-11T03:42:04,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742302_1478 (size=12151) 2024-11-11T03:42:04,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/248a1bc246c84c649f968f550ff5b67e 2024-11-11T03:42:04,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/d3dbdd7155184abaa0eea7c50ecfb872 is 50, key is test_row_0/C:col10/1731296524298/Put/seqid=0 2024-11-11T03:42:04,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742303_1479 (size=12151) 2024-11-11T03:42:04,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:04,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:04,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:04,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:04,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296584982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296584982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:05,089 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/4702dbd912a241e6bd64c03e80a0b1bb as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/4702dbd912a241e6bd64c03e80a0b1bb 2024-11-11T03:42:05,092 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/C of e07956485faf1a58beb177dd117c82c8 into 4702dbd912a241e6bd64c03e80a0b1bb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:05,092 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:05,092 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/C, priority=13, startTime=1731296524252; duration=0sec 2024-11-11T03:42:05,092 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:05,092 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:42:05,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/d3dbdd7155184abaa0eea7c50ecfb872 2024-11-11T03:42:05,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a36a317442f6409e93a844c52ef666b6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a36a317442f6409e93a844c52ef666b6 2024-11-11T03:42:05,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a36a317442f6409e93a844c52ef666b6, entries=200, sequenceid=245, filesize=14.2 K 2024-11-11T03:42:05,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/248a1bc246c84c649f968f550ff5b67e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/248a1bc246c84c649f968f550ff5b67e 2024-11-11T03:42:05,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/248a1bc246c84c649f968f550ff5b67e, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T03:42:05,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/d3dbdd7155184abaa0eea7c50ecfb872 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/d3dbdd7155184abaa0eea7c50ecfb872 2024-11-11T03:42:05,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/d3dbdd7155184abaa0eea7c50ecfb872, entries=150, sequenceid=245, filesize=11.9 K 2024-11-11T03:42:05,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T03:42:05,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for e07956485faf1a58beb177dd117c82c8 in 839ms, sequenceid=245, compaction requested=false 2024-11-11T03:42:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:05,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:05,138 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:42:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:05,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/55e4416ba08341318bb3231e04c6b44b is 50, key is test_row_0/A:col10/1731296524334/Put/seqid=0 2024-11-11T03:42:05,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742304_1480 (size=12151) 2024-11-11T03:42:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:05,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296525319, max timestamp=1731296525353Num uniques 859; 2024-11-11T03:42:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296525319, max timestamp=1731296525353Num uniques 859; 2024-11-11T03:42:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296525319, max timestamp=1731296525353Num uniques 859; 2024-11-11T03:42:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:05,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T03:42:05,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296585498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:05,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:05,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296585502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:05,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296585502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:05,546 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/55e4416ba08341318bb3231e04c6b44b 2024-11-11T03:42:05,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/b50ffa386e9c486f91e64062e7498370 is 50, key is test_row_0/B:col10/1731296524334/Put/seqid=0 2024-11-11T03:42:05,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742305_1481 (size=12151) 2024-11-11T03:42:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296585612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296585832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:05,953 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/b50ffa386e9c486f91e64062e7498370 2024-11-11T03:42:05,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/58270f62e1894e019f5bd869533cd07f is 50, key is test_row_0/C:col10/1731296524334/Put/seqid=0 2024-11-11T03:42:05,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742306_1482 (size=12151) 2024-11-11T03:42:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296586152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,362 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/58270f62e1894e019f5bd869533cd07f 2024-11-11T03:42:06,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/55e4416ba08341318bb3231e04c6b44b as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/55e4416ba08341318bb3231e04c6b44b 2024-11-11T03:42:06,368 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/55e4416ba08341318bb3231e04c6b44b, entries=150, sequenceid=258, filesize=11.9 K 2024-11-11T03:42:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/b50ffa386e9c486f91e64062e7498370 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b50ffa386e9c486f91e64062e7498370 2024-11-11T03:42:06,371 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b50ffa386e9c486f91e64062e7498370, entries=150, sequenceid=258, filesize=11.9 K 2024-11-11T03:42:06,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/58270f62e1894e019f5bd869533cd07f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/58270f62e1894e019f5bd869533cd07f 2024-11-11T03:42:06,374 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/58270f62e1894e019f5bd869533cd07f, entries=150, sequenceid=258, filesize=11.9 K 2024-11-11T03:42:06,374 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e07956485faf1a58beb177dd117c82c8 in 1236ms, sequenceid=258, compaction requested=true 2024-11-11T03:42:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:06,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-11T03:42:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-11T03:42:06,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T03:42:06,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0710 sec 2024-11-11T03:42:06,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 3.0750 sec 2024-11-11T03:42:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:06,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T03:42:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:42:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:42:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:42:06,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a455e6e4cc2849c9950c3b8e44210bb1 is 50, key is test_row_0/A:col10/1731296525487/Put/seqid=0 2024-11-11T03:42:06,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742307_1483 (size=14741) 2024-11-11T03:42:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296586563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296586565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296586675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296586675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296586681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296586892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296586892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:06,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a455e6e4cc2849c9950c3b8e44210bb1 2024-11-11T03:42:06,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/1b77614d29794d2898972f7d2e9881c3 is 50, key is test_row_0/B:col10/1731296525487/Put/seqid=0 2024-11-11T03:42:06,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742308_1484 (size=12301) 2024-11-11T03:42:06,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/1b77614d29794d2898972f7d2e9881c3 2024-11-11T03:42:06,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/bb2b4c37ea8649fab5b74425f7c62787 is 50, key is test_row_0/C:col10/1731296525487/Put/seqid=0 2024-11-11T03:42:06,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742309_1485 (size=12301) 2024-11-11T03:42:07,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296587211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296587211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:07,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/bb2b4c37ea8649fab5b74425f7c62787 2024-11-11T03:42:07,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/a455e6e4cc2849c9950c3b8e44210bb1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a455e6e4cc2849c9950c3b8e44210bb1 2024-11-11T03:42:07,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a455e6e4cc2849c9950c3b8e44210bb1, entries=200, sequenceid=296, filesize=14.4 K 2024-11-11T03:42:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/1b77614d29794d2898972f7d2e9881c3 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1b77614d29794d2898972f7d2e9881c3 2024-11-11T03:42:07,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1b77614d29794d2898972f7d2e9881c3, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T03:42:07,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/bb2b4c37ea8649fab5b74425f7c62787 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/bb2b4c37ea8649fab5b74425f7c62787 2024-11-11T03:42:07,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/bb2b4c37ea8649fab5b74425f7c62787, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T03:42:07,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e07956485faf1a58beb177dd117c82c8 in 840ms, sequenceid=296, compaction requested=true 2024-11-11T03:42:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:07,372 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:42:07,372 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:42:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:07,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:07,373 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:42:07,373 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/A is initiating minor compaction (all files) 2024-11-11T03:42:07,373 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/A in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:07,374 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/753e884281e34f99a1a80a1fc543cdb4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a36a317442f6409e93a844c52ef666b6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/55e4416ba08341318bb3231e04c6b44b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a455e6e4cc2849c9950c3b8e44210bb1] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=52.8 K 2024-11-11T03:42:07,374 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:42:07,374 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/B is initiating minor compaction (all files) 2024-11-11T03:42:07,374 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/B in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:07,374 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/46747eedc4064a7e94faf3c95905911d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/248a1bc246c84c649f968f550ff5b67e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b50ffa386e9c486f91e64062e7498370, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1b77614d29794d2898972f7d2e9881c3] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=48.1 K 2024-11-11T03:42:07,374 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 753e884281e34f99a1a80a1fc543cdb4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731296522365 2024-11-11T03:42:07,374 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 46747eedc4064a7e94faf3c95905911d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731296522365 2024-11-11T03:42:07,374 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36a317442f6409e93a844c52ef666b6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731296523108 2024-11-11T03:42:07,374 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 248a1bc246c84c649f968f550ff5b67e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731296523108 2024-11-11T03:42:07,375 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b50ffa386e9c486f91e64062e7498370, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731296524314 2024-11-11T03:42:07,375 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e4416ba08341318bb3231e04c6b44b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731296524314 2024-11-11T03:42:07,375 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting a455e6e4cc2849c9950c3b8e44210bb1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731296525475 2024-11-11T03:42:07,375 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b77614d29794d2898972f7d2e9881c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731296525487 2024-11-11T03:42:07,382 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#B#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:07,383 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/aac1a38362a342ddb0bb60b15891650c is 50, key is test_row_0/B:col10/1731296525487/Put/seqid=0 2024-11-11T03:42:07,387 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#A#compaction#414 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:07,387 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/504662adb5434d9da61dc472451c642c is 50, key is test_row_0/A:col10/1731296525487/Put/seqid=0 2024-11-11T03:42:07,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742310_1486 (size=12949) 2024-11-11T03:42:07,396 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/aac1a38362a342ddb0bb60b15891650c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/aac1a38362a342ddb0bb60b15891650c 2024-11-11T03:42:07,402 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e07956485faf1a58beb177dd117c82c8/B of e07956485faf1a58beb177dd117c82c8 into aac1a38362a342ddb0bb60b15891650c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:07,402 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:07,402 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/B, priority=12, startTime=1731296527372; duration=0sec 2024-11-11T03:42:07,402 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:07,402 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:42:07,402 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T03:42:07,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742311_1487 (size=12949) 2024-11-11T03:42:07,403 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T03:42:07,403 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/C is initiating minor compaction (all files) 2024-11-11T03:42:07,403 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/C in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:07,403 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/4702dbd912a241e6bd64c03e80a0b1bb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/d3dbdd7155184abaa0eea7c50ecfb872, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/58270f62e1894e019f5bd869533cd07f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/bb2b4c37ea8649fab5b74425f7c62787] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=48.1 K 2024-11-11T03:42:07,404 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4702dbd912a241e6bd64c03e80a0b1bb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731296522365 2024-11-11T03:42:07,404 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting d3dbdd7155184abaa0eea7c50ecfb872, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731296523108 2024-11-11T03:42:07,404 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 58270f62e1894e019f5bd869533cd07f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731296524314 2024-11-11T03:42:07,404 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting bb2b4c37ea8649fab5b74425f7c62787, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731296525487 2024-11-11T03:42:07,408 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/504662adb5434d9da61dc472451c642c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/504662adb5434d9da61dc472451c642c 2024-11-11T03:42:07,414 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e07956485faf1a58beb177dd117c82c8/A of e07956485faf1a58beb177dd117c82c8 into 504662adb5434d9da61dc472451c642c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:07,414 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:07,414 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/A, priority=12, startTime=1731296527372; duration=0sec 2024-11-11T03:42:07,414 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:07,414 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:42:07,417 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#C#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:07,417 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/fcc802a3582d4e8d836568b5f3f6a8fc is 50, key is test_row_0/C:col10/1731296525487/Put/seqid=0 2024-11-11T03:42:07,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742312_1488 (size=12949) 2024-11-11T03:42:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T03:42:07,462 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:07,462 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-11T03:42:07,464 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T03:42:07,465 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:07,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T03:42:07,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T03:42:07,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:07,617 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T03:42:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:07,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/e35ee79bda374cd3bec26b68d954773d is 50, key is test_row_0/A:col10/1731296526561/Put/seqid=0 2024-11-11T03:42:07,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742313_1489 (size=9857) 2024-11-11T03:42:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:07,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296527753, max timestamp=1731296527770Num uniques 859; 2024-11-11T03:42:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296527753, max timestamp=1731296527770Num uniques 859; 2024-11-11T03:42:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731296527753, max timestamp=1731296527770Num uniques 859; 2024-11-11T03:42:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:42:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:42:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T03:42:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T03:42:07,823 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/fcc802a3582d4e8d836568b5f3f6a8fc as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/fcc802a3582d4e8d836568b5f3f6a8fc 2024-11-11T03:42:07,826 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e07956485faf1a58beb177dd117c82c8/C of e07956485faf1a58beb177dd117c82c8 into fcc802a3582d4e8d836568b5f3f6a8fc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:07,826 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:07,826 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/C, priority=12, startTime=1731296527372; duration=0sec 2024-11-11T03:42:07,827 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:07,827 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:42:07,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296587820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:07,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296587827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:07,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296587828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:07,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296587942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296587942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296587942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:08,035 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/e35ee79bda374cd3bec26b68d954773d 2024-11-11T03:42:08,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/91c17e1054104aaba9e426fd04b87098 is 50, key is test_row_0/B:col10/1731296526561/Put/seqid=0 2024-11-11T03:42:08,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742314_1490 (size=9857) 2024-11-11T03:42:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T03:42:08,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296588162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:08,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296588162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:08,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296588162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:08,443 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/91c17e1054104aaba9e426fd04b87098 2024-11-11T03:42:08,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/e71ce060e69241d092ac9a407c4773df is 50, key is test_row_0/C:col10/1731296526561/Put/seqid=0 2024-11-11T03:42:08,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742315_1491 (size=9857) 2024-11-11T03:42:08,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296588471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296588472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296588472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:08,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T03:42:08,852 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/e71ce060e69241d092ac9a407c4773df 2024-11-11T03:42:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/e35ee79bda374cd3bec26b68d954773d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/e35ee79bda374cd3bec26b68d954773d 2024-11-11T03:42:08,857 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/e35ee79bda374cd3bec26b68d954773d, entries=100, sequenceid=307, filesize=9.6 K 2024-11-11T03:42:08,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/91c17e1054104aaba9e426fd04b87098 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/91c17e1054104aaba9e426fd04b87098 2024-11-11T03:42:08,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/91c17e1054104aaba9e426fd04b87098, entries=100, sequenceid=307, filesize=9.6 K 2024-11-11T03:42:08,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/e71ce060e69241d092ac9a407c4773df as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/e71ce060e69241d092ac9a407c4773df 2024-11-11T03:42:08,864 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/e71ce060e69241d092ac9a407c4773df, entries=100, sequenceid=307, filesize=9.6 K 2024-11-11T03:42:08,864 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for e07956485faf1a58beb177dd117c82c8 in 1247ms, sequenceid=307, compaction requested=false 2024-11-11T03:42:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:08,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-11T03:42:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-11T03:42:08,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T03:42:08,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-11T03:42:08,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4050 sec 2024-11-11T03:42:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:08,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T03:42:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:42:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:42:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T03:42:08,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/fe0dab8261c24ccdbd53531b59894887 is 50, key is test_row_0/A:col10/1731296527825/Put/seqid=0 2024-11-11T03:42:08,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742316_1492 (size=14741) 2024-11-11T03:42:08,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/fe0dab8261c24ccdbd53531b59894887 2024-11-11T03:42:09,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/333884646c474f438077573c76a27ae9 is 50, key is test_row_0/B:col10/1731296527825/Put/seqid=0 2024-11-11T03:42:09,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742317_1493 (size=12301) 2024-11-11T03:42:09,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296589006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296589007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296589010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296589122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296589132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296589132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296589342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296589352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296589352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/333884646c474f438077573c76a27ae9 2024-11-11T03:42:09,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/cf9913591cfc49a1b91e7c97685419ea is 50, key is test_row_0/C:col10/1731296527825/Put/seqid=0 2024-11-11T03:42:09,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742318_1494 (size=12301) 2024-11-11T03:42:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T03:42:09,621 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:09,622 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-11T03:42:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T03:42:09,623 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:09,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:09,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:09,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296589652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296589662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296589662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T03:42:09,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T03:42:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:09,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:09,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/cf9913591cfc49a1b91e7c97685419ea 2024-11-11T03:42:09,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/fe0dab8261c24ccdbd53531b59894887 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/fe0dab8261c24ccdbd53531b59894887 2024-11-11T03:42:09,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/fe0dab8261c24ccdbd53531b59894887, entries=200, sequenceid=346, filesize=14.4 K 2024-11-11T03:42:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/333884646c474f438077573c76a27ae9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/333884646c474f438077573c76a27ae9 2024-11-11T03:42:09,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/333884646c474f438077573c76a27ae9, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T03:42:09,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/cf9913591cfc49a1b91e7c97685419ea as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/cf9913591cfc49a1b91e7c97685419ea 2024-11-11T03:42:09,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/cf9913591cfc49a1b91e7c97685419ea, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T03:42:09,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for e07956485faf1a58beb177dd117c82c8 in 838ms, sequenceid=346, compaction requested=true 2024-11-11T03:42:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:09,825 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:09,825 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:09,826 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:09,826 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:09,826 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/A is initiating minor compaction (all files) 2024-11-11T03:42:09,826 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/B is initiating minor compaction (all files) 2024-11-11T03:42:09,826 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/B in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:09,826 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/A in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:09,826 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/504662adb5434d9da61dc472451c642c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/e35ee79bda374cd3bec26b68d954773d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/fe0dab8261c24ccdbd53531b59894887] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=36.7 K 2024-11-11T03:42:09,826 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/aac1a38362a342ddb0bb60b15891650c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/91c17e1054104aaba9e426fd04b87098, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/333884646c474f438077573c76a27ae9] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=34.3 K 2024-11-11T03:42:09,826 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 504662adb5434d9da61dc472451c642c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731296525487 2024-11-11T03:42:09,826 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting aac1a38362a342ddb0bb60b15891650c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731296525487 2024-11-11T03:42:09,827 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c17e1054104aaba9e426fd04b87098, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731296526561 2024-11-11T03:42:09,827 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e35ee79bda374cd3bec26b68d954773d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731296526561 2024-11-11T03:42:09,827 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 333884646c474f438077573c76a27ae9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731296527825 2024-11-11T03:42:09,827 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0dab8261c24ccdbd53531b59894887, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731296527812 2024-11-11T03:42:09,832 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:09,832 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/2f7d03785d8f429898fb6a5c3339df78 is 50, key is test_row_0/A:col10/1731296527825/Put/seqid=0 2024-11-11T03:42:09,832 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:09,832 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/e5f1d087d43e4c94a64cf968a83f8a08 is 50, key is test_row_0/B:col10/1731296527825/Put/seqid=0 2024-11-11T03:42:09,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742320_1496 (size=13051) 2024-11-11T03:42:09,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742319_1495 (size=13051) 2024-11-11T03:42:09,840 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/2f7d03785d8f429898fb6a5c3339df78 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2f7d03785d8f429898fb6a5c3339df78 2024-11-11T03:42:09,840 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/e5f1d087d43e4c94a64cf968a83f8a08 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/e5f1d087d43e4c94a64cf968a83f8a08 2024-11-11T03:42:09,844 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/A of e07956485faf1a58beb177dd117c82c8 into 2f7d03785d8f429898fb6a5c3339df78(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:09,844 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/B of e07956485faf1a58beb177dd117c82c8 into e5f1d087d43e4c94a64cf968a83f8a08(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:09,844 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:09,844 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:09,844 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/B, priority=13, startTime=1731296529825; duration=0sec 2024-11-11T03:42:09,844 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/A, priority=13, startTime=1731296529825; duration=0sec 2024-11-11T03:42:09,844 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:09,844 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:09,844 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:42:09,845 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:42:09,845 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:09,847 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:09,847 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): e07956485faf1a58beb177dd117c82c8/C is initiating minor compaction (all files) 2024-11-11T03:42:09,847 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e07956485faf1a58beb177dd117c82c8/C in TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:09,847 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/fcc802a3582d4e8d836568b5f3f6a8fc, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/e71ce060e69241d092ac9a407c4773df, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/cf9913591cfc49a1b91e7c97685419ea] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp, totalSize=34.3 K 2024-11-11T03:42:09,847 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc802a3582d4e8d836568b5f3f6a8fc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731296525487 2024-11-11T03:42:09,848 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e71ce060e69241d092ac9a407c4773df, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731296526561 2024-11-11T03:42:09,848 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9913591cfc49a1b91e7c97685419ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731296527825 2024-11-11T03:42:09,853 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e07956485faf1a58beb177dd117c82c8#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:09,854 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/1525ff4a00244b729ef0545122c425ea is 50, key is test_row_0/C:col10/1731296527825/Put/seqid=0 2024-11-11T03:42:09,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742321_1497 (size=13051) 2024-11-11T03:42:09,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T03:42:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:09,929 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T03:42:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:09,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/41cdcd63f9524a8ea670c86b9d8271bf is 50, key is test_row_0/A:col10/1731296528989/Put/seqid=0 2024-11-11T03:42:09,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742322_1498 (size=12301) 2024-11-11T03:42:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T03:42:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:10,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. as already flushing 2024-11-11T03:42:10,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296530107, max timestamp=1731296530129Num uniques 859; 2024-11-11T03:42:10,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296530107, max timestamp=1731296530129Num uniques 859; 2024-11-11T03:42:10,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:10,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:10,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296530107, max timestamp=1731296530129Num uniques 859; 2024-11-11T03:42:10,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:10,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296590184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731296530177, max timestamp=1731296530196Num uniques 1018; 2024-11-11T03:42:10,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T03:42:10,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731296530177, max timestamp=1731296530196Num uniques 1018; 2024-11-11T03:42:10,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731296530177, max timestamp=1731296530196Num uniques 1018; 2024-11-11T03:42:10,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T03:42:10,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T03:42:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296590245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731296530196, max timestamp=1731296530248Num uniques 1077; 2024-11-11T03:42:10,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T03:42:10,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731296530196, max timestamp=1731296530248Num uniques 1077; 2024-11-11T03:42:10,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731296530196, max timestamp=1731296530248Num uniques 1077; 2024-11-11T03:42:10,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T03:42:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T03:42:10,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-11T03:42:10,262 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/1525ff4a00244b729ef0545122c425ea as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/1525ff4a00244b729ef0545122c425ea 2024-11-11T03:42:10,266 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e07956485faf1a58beb177dd117c82c8/C of e07956485faf1a58beb177dd117c82c8 into 1525ff4a00244b729ef0545122c425ea(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:10,266 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:10,266 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., storeName=e07956485faf1a58beb177dd117c82c8/C, priority=13, startTime=1731296529825; duration=0sec 2024-11-11T03:42:10,266 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:10,266 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:42:10,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296590312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731296530291, max timestamp=1731296530319Num uniques 1136; 2024-11-11T03:42:10,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-11T03:42:10,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731296530291, max timestamp=1731296530319Num uniques 1136; 2024-11-11T03:42:10,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731296530291, max timestamp=1731296530319Num uniques 1136; 2024-11-11T03:42:10,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-11T03:42:10,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-11T03:42:10,336 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/41cdcd63f9524a8ea670c86b9d8271bf 2024-11-11T03:42:10,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ba44024d68cf4e0e8523da70e5ec0880 is 50, key is test_row_0/B:col10/1731296528989/Put/seqid=0 2024-11-11T03:42:10,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742323_1499 (size=12301) 2024-11-11T03:42:10,348 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ba44024d68cf4e0e8523da70e5ec0880 2024-11-11T03:42:10,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/9a3db68215394a7bb1e7a75772a04159 is 50, key is test_row_0/C:col10/1731296528989/Put/seqid=0 2024-11-11T03:42:10,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742324_1500 (size=12301) 2024-11-11T03:42:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296590378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731296530352, max timestamp=1731296530389Num uniques 1145; 2024-11-11T03:42:10,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-11T03:42:10,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731296530352, max timestamp=1731296530389Num uniques 1145; 2024-11-11T03:42:10,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-11T03:42:10,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731296530352, max timestamp=1731296530389Num uniques 1145; 2024-11-11T03:42:10,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-11T03:42:10,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296590446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296590447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296590461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296590501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39416 deadline: 1731296590532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,538 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-11-11T03:42:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,538 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:10,538 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,539 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:10,540 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-11-11T03:42:10,540 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:10,540 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,540 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:10,549 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-11-11T03:42:10,549 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:10,549 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,549 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,549 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:10,550 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-11-11T03:42:10,550 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:10,550 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,551 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:10,553 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-11T03:42:10,553 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:10,553 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,553 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296590572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296590572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39474 deadline: 1731296590582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39484 deadline: 1731296590722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,759 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/9a3db68215394a7bb1e7a75772a04159 2024-11-11T03:42:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T03:42:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/41cdcd63f9524a8ea670c86b9d8271bf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/41cdcd63f9524a8ea670c86b9d8271bf 2024-11-11T03:42:10,773 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/41cdcd63f9524a8ea670c86b9d8271bf, entries=150, sequenceid=356, filesize=12.0 K 2024-11-11T03:42:10,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/ba44024d68cf4e0e8523da70e5ec0880 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba44024d68cf4e0e8523da70e5ec0880 2024-11-11T03:42:10,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba44024d68cf4e0e8523da70e5ec0880, entries=150, sequenceid=356, filesize=12.0 K 2024-11-11T03:42:10,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/9a3db68215394a7bb1e7a75772a04159 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/9a3db68215394a7bb1e7a75772a04159 2024-11-11T03:42:10,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39460 deadline: 1731296590782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1731296590782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 , the old value is region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=2 because the exception is null or not the one we care about 2024-11-11T03:42:10,784 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/9a3db68215394a7bb1e7a75772a04159, entries=150, sequenceid=356, filesize=12.0 K 2024-11-11T03:42:10,785 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for e07956485faf1a58beb177dd117c82c8 in 856ms, sequenceid=356, compaction requested=false 2024-11-11T03:42:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-11T03:42:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-11T03:42:10,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T03:42:10,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1630 sec 2024-11-11T03:42:10,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.1690 sec 2024-11-11T03:42:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:10,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-11T03:42:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:10,794 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-11-11T03:42:10,794 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:10,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:10,795 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:10,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:10,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:10,795 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:10,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:10,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/71f7e54bc4c84fcea0bb9487a61c2c08 is 50, key is test_row_0/A:col10/1731296530792/Put/seqid=0 2024-11-11T03:42:10,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742325_1501 (size=12301) 2024-11-11T03:42:10,843 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-11-11T03:42:10,844 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:10,844 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,844 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:10,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:11,036 INFO [Thread-1935 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1935. 2024-11-11T03:42:11,037 DEBUG [Thread-1935 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:11,037 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,037 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:11,098 INFO [Thread-1933 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1933. 2024-11-11T03:42:11,098 INFO [Thread-1931 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1931. 2024-11-11T03:42:11,098 DEBUG [Thread-1933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:11,098 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,098 DEBUG [Thread-1931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:11,098 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,098 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:11,099 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:11,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/71f7e54bc4c84fcea0bb9487a61c2c08 2024-11-11T03:42:11,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f6e35206f62148babbe7b6b2bdce5aff is 50, key is test_row_0/B:col10/1731296530792/Put/seqid=0 2024-11-11T03:42:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742326_1502 (size=12301) 2024-11-11T03:42:11,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f6e35206f62148babbe7b6b2bdce5aff 2024-11-11T03:42:11,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/56a19f8c82474809815c63ab7fc8a125 is 50, key is test_row_0/C:col10/1731296530792/Put/seqid=0 2024-11-11T03:42:11,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742327_1503 (size=12301) 2024-11-11T03:42:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T03:42:11,773 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:11,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T03:42:11,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-11T03:42:11,773 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3502 rows 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1158 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3469 rows 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1153 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3459 rows 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3466 rows 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1158 2024-11-11T03:42:11,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3472 rows 2024-11-11T03:42:11,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:42:11,775 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:42:11,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:11,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:11,777 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T03:42:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T03:42:11,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296531782"}]},"ts":"1731296531782"} 2024-11-11T03:42:11,784 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T03:42:11,784 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T03:42:11,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:42:11,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, UNASSIGN}] 2024-11-11T03:42:11,787 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, UNASSIGN 2024-11-11T03:42:11,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e07956485faf1a58beb177dd117c82c8, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:11,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, UNASSIGN because future has completed 2024-11-11T03:42:11,789 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:42:11,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:42:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T03:42:11,941 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:11,941 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:42:11,942 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing e07956485faf1a58beb177dd117c82c8, disabling compactions & flushes 2024-11-11T03:42:11,942 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:12,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/56a19f8c82474809815c63ab7fc8a125 2024-11-11T03:42:12,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/71f7e54bc4c84fcea0bb9487a61c2c08 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71f7e54bc4c84fcea0bb9487a61c2c08 2024-11-11T03:42:12,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71f7e54bc4c84fcea0bb9487a61c2c08, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T03:42:12,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/f6e35206f62148babbe7b6b2bdce5aff as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f6e35206f62148babbe7b6b2bdce5aff 2024-11-11T03:42:12,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f6e35206f62148babbe7b6b2bdce5aff, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T03:42:12,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/56a19f8c82474809815c63ab7fc8a125 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/56a19f8c82474809815c63ab7fc8a125 2024-11-11T03:42:12,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/56a19f8c82474809815c63ab7fc8a125, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T03:42:12,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for e07956485faf1a58beb177dd117c82c8 in 1268ms, sequenceid=463, compaction requested=true 2024-11-11T03:42:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e07956485faf1a58beb177dd117c82c8: 2024-11-11T03:42:12,062 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. after waiting 0 ms 2024-11-11T03:42:12,062 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. because compaction request was cancelled 2024-11-11T03:42:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:12,062 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:A 2024-11-11T03:42:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:12,062 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. because compaction request was cancelled 2024-11-11T03:42:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e07956485faf1a58beb177dd117c82c8:C, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:12,062 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing e07956485faf1a58beb177dd117c82c8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T03:42:12,062 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:B 2024-11-11T03:42:12,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:12,062 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. because compaction request was cancelled 2024-11-11T03:42:12,062 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e07956485faf1a58beb177dd117c82c8:C 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=A 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=B 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e07956485faf1a58beb177dd117c82c8, store=C 2024-11-11T03:42:12,062 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:12,065 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/7e50919d25884e7187b734865f5e914f is 50, key is test_row_0/A:col10/1731296531093/Put/seqid=0 2024-11-11T03:42:12,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742328_1504 (size=9857) 2024-11-11T03:42:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T03:42:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T03:42:12,469 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/7e50919d25884e7187b734865f5e914f 2024-11-11T03:42:12,482 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/8769276ddb224f379ad509a1f46e3ec2 is 50, key is test_row_0/B:col10/1731296531093/Put/seqid=0 2024-11-11T03:42:12,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742329_1505 (size=9857) 2024-11-11T03:42:12,622 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:42:12,887 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/8769276ddb224f379ad509a1f46e3ec2 2024-11-11T03:42:12,900 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/efdf84aa9ecc40729ac890930731830a is 50, key is test_row_0/C:col10/1731296531093/Put/seqid=0 2024-11-11T03:42:12,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742330_1506 (size=9857) 2024-11-11T03:42:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T03:42:13,307 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/efdf84aa9ecc40729ac890930731830a 2024-11-11T03:42:13,316 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/A/7e50919d25884e7187b734865f5e914f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7e50919d25884e7187b734865f5e914f 2024-11-11T03:42:13,320 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7e50919d25884e7187b734865f5e914f, entries=100, sequenceid=470, filesize=9.6 K 2024-11-11T03:42:13,321 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/B/8769276ddb224f379ad509a1f46e3ec2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8769276ddb224f379ad509a1f46e3ec2 2024-11-11T03:42:13,324 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8769276ddb224f379ad509a1f46e3ec2, entries=100, sequenceid=470, filesize=9.6 K 2024-11-11T03:42:13,325 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/.tmp/C/efdf84aa9ecc40729ac890930731830a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/efdf84aa9ecc40729ac890930731830a 2024-11-11T03:42:13,329 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/efdf84aa9ecc40729ac890930731830a, entries=100, sequenceid=470, filesize=9.6 K 2024-11-11T03:42:13,330 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e07956485faf1a58beb177dd117c82c8 in 1267ms, sequenceid=470, compaction requested=true 2024-11-11T03:42:13,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a852c2dd5c784bcf9e13fa74ea3a04de, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71c92f10bd0f4328b6709b64ba8c0523, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7d3e1cbb64d14c80b9f2c977171d7763, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/73cbc213c3a143f4ac226f871edbcc03, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/ac0f5ac1fbe74190ae09fc20346da38a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/98022d3f2a6a43929ccc2a2dfe861635, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/410a1f9f6a094d988ecda51847b066bf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/d2a2a623bd00453c9e4265c9aa90b032, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a1195a592de042f3939fdfce2a4d76fe, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/0a9f7af96af34161a0ffbfbc9dce48d6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/14ba7b1a53ca4d01b045b137d51c1098, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/bebe273dec9848bf9640b8cf415a5c44, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/44c349ca263b41008dff7e79b82a9f46, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2d3dfe24e1c04ae68b881b8cec7f786f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/be840fa0035843b989ede00cc0c131e7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/753e884281e34f99a1a80a1fc543cdb4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a36a317442f6409e93a844c52ef666b6, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/55e4416ba08341318bb3231e04c6b44b, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a455e6e4cc2849c9950c3b8e44210bb1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/504662adb5434d9da61dc472451c642c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/e35ee79bda374cd3bec26b68d954773d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/fe0dab8261c24ccdbd53531b59894887] to archive 2024-11-11T03:42:13,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:42:13,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a852c2dd5c784bcf9e13fa74ea3a04de to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a852c2dd5c784bcf9e13fa74ea3a04de 2024-11-11T03:42:13,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71c92f10bd0f4328b6709b64ba8c0523 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71c92f10bd0f4328b6709b64ba8c0523 2024-11-11T03:42:13,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7d3e1cbb64d14c80b9f2c977171d7763 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7d3e1cbb64d14c80b9f2c977171d7763 2024-11-11T03:42:13,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/73cbc213c3a143f4ac226f871edbcc03 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/73cbc213c3a143f4ac226f871edbcc03 2024-11-11T03:42:13,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/ac0f5ac1fbe74190ae09fc20346da38a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/ac0f5ac1fbe74190ae09fc20346da38a 2024-11-11T03:42:13,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/98022d3f2a6a43929ccc2a2dfe861635 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/98022d3f2a6a43929ccc2a2dfe861635 2024-11-11T03:42:13,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/410a1f9f6a094d988ecda51847b066bf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/410a1f9f6a094d988ecda51847b066bf 2024-11-11T03:42:13,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/d2a2a623bd00453c9e4265c9aa90b032 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/d2a2a623bd00453c9e4265c9aa90b032 2024-11-11T03:42:13,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a1195a592de042f3939fdfce2a4d76fe to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a1195a592de042f3939fdfce2a4d76fe 2024-11-11T03:42:13,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/0a9f7af96af34161a0ffbfbc9dce48d6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/0a9f7af96af34161a0ffbfbc9dce48d6 2024-11-11T03:42:13,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/14ba7b1a53ca4d01b045b137d51c1098 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/14ba7b1a53ca4d01b045b137d51c1098 2024-11-11T03:42:13,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/bebe273dec9848bf9640b8cf415a5c44 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/bebe273dec9848bf9640b8cf415a5c44 2024-11-11T03:42:13,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/44c349ca263b41008dff7e79b82a9f46 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/44c349ca263b41008dff7e79b82a9f46 2024-11-11T03:42:13,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2d3dfe24e1c04ae68b881b8cec7f786f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2d3dfe24e1c04ae68b881b8cec7f786f 2024-11-11T03:42:13,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/be840fa0035843b989ede00cc0c131e7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/be840fa0035843b989ede00cc0c131e7 2024-11-11T03:42:13,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/753e884281e34f99a1a80a1fc543cdb4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/753e884281e34f99a1a80a1fc543cdb4 2024-11-11T03:42:13,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a36a317442f6409e93a844c52ef666b6 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a36a317442f6409e93a844c52ef666b6 2024-11-11T03:42:13,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/55e4416ba08341318bb3231e04c6b44b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/55e4416ba08341318bb3231e04c6b44b 2024-11-11T03:42:13,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a455e6e4cc2849c9950c3b8e44210bb1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/a455e6e4cc2849c9950c3b8e44210bb1 2024-11-11T03:42:13,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/504662adb5434d9da61dc472451c642c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/504662adb5434d9da61dc472451c642c 2024-11-11T03:42:13,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/e35ee79bda374cd3bec26b68d954773d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/e35ee79bda374cd3bec26b68d954773d 2024-11-11T03:42:13,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/fe0dab8261c24ccdbd53531b59894887 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/fe0dab8261c24ccdbd53531b59894887 2024-11-11T03:42:13,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8e1a7eb9c60c4e0e99de0d0506025636, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/fec56d764c2b4251804fdab8a80af448, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/300ad79011854deda2f4edba3ed79dfd, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/6649702cff0c4f579816e805a6f4744f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1660e08536ae4dc884804c2d1649c459, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b0ac2aaf7edd4af0b0b86f39118c5e40, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/3390279e9ea546e7a34e779787e0275a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/53f8338a3bae4a2eb57a571861d65f0e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba50621893de4f86b7c14c695fd316c1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f27c53ffc13c43d1b92d90ecc935a227, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ce10d09f6bbc48c8afc3c81458eeff7c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/60bee191f02b4a389a222c11e9cfe6c2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/5fd5fddc15b44e0985837f6e7359f2d4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/785d4cb41fa24f489d644ea6ef79c420, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/46747eedc4064a7e94faf3c95905911d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f0b2f80050264a8a8ce4a6e434b12090, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/248a1bc246c84c649f968f550ff5b67e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b50ffa386e9c486f91e64062e7498370, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/aac1a38362a342ddb0bb60b15891650c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1b77614d29794d2898972f7d2e9881c3, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/91c17e1054104aaba9e426fd04b87098, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/333884646c474f438077573c76a27ae9] to archive 2024-11-11T03:42:13,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:42:13,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8e1a7eb9c60c4e0e99de0d0506025636 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8e1a7eb9c60c4e0e99de0d0506025636 2024-11-11T03:42:13,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/fec56d764c2b4251804fdab8a80af448 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/fec56d764c2b4251804fdab8a80af448 2024-11-11T03:42:13,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/300ad79011854deda2f4edba3ed79dfd to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/300ad79011854deda2f4edba3ed79dfd 2024-11-11T03:42:13,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/6649702cff0c4f579816e805a6f4744f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/6649702cff0c4f579816e805a6f4744f 2024-11-11T03:42:13,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1660e08536ae4dc884804c2d1649c459 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1660e08536ae4dc884804c2d1649c459 2024-11-11T03:42:13,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b0ac2aaf7edd4af0b0b86f39118c5e40 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b0ac2aaf7edd4af0b0b86f39118c5e40 2024-11-11T03:42:13,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/3390279e9ea546e7a34e779787e0275a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/3390279e9ea546e7a34e779787e0275a 2024-11-11T03:42:13,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/53f8338a3bae4a2eb57a571861d65f0e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/53f8338a3bae4a2eb57a571861d65f0e 2024-11-11T03:42:13,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba50621893de4f86b7c14c695fd316c1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba50621893de4f86b7c14c695fd316c1 2024-11-11T03:42:13,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f27c53ffc13c43d1b92d90ecc935a227 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f27c53ffc13c43d1b92d90ecc935a227 2024-11-11T03:42:13,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ce10d09f6bbc48c8afc3c81458eeff7c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ce10d09f6bbc48c8afc3c81458eeff7c 2024-11-11T03:42:13,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/60bee191f02b4a389a222c11e9cfe6c2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/60bee191f02b4a389a222c11e9cfe6c2 2024-11-11T03:42:13,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/5fd5fddc15b44e0985837f6e7359f2d4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/5fd5fddc15b44e0985837f6e7359f2d4 2024-11-11T03:42:13,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/785d4cb41fa24f489d644ea6ef79c420 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/785d4cb41fa24f489d644ea6ef79c420 2024-11-11T03:42:13,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/46747eedc4064a7e94faf3c95905911d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/46747eedc4064a7e94faf3c95905911d 2024-11-11T03:42:13,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f0b2f80050264a8a8ce4a6e434b12090 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f0b2f80050264a8a8ce4a6e434b12090 2024-11-11T03:42:13,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/248a1bc246c84c649f968f550ff5b67e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/248a1bc246c84c649f968f550ff5b67e 2024-11-11T03:42:13,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b50ffa386e9c486f91e64062e7498370 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/b50ffa386e9c486f91e64062e7498370 2024-11-11T03:42:13,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/aac1a38362a342ddb0bb60b15891650c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/aac1a38362a342ddb0bb60b15891650c 2024-11-11T03:42:13,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1b77614d29794d2898972f7d2e9881c3 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/1b77614d29794d2898972f7d2e9881c3 2024-11-11T03:42:13,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/91c17e1054104aaba9e426fd04b87098 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/91c17e1054104aaba9e426fd04b87098 2024-11-11T03:42:13,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/333884646c474f438077573c76a27ae9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/333884646c474f438077573c76a27ae9 2024-11-11T03:42:13,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/82223b7cdf8947b6b04227fc77696299, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/31e180ea5f234bc99ce8b29892d3b784, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/81b0b1e036de40e09fd958b7c8d0fa8d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/c421be37e4b747dbb852f2bd8c387ecb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/93c2d21563a8480c9cf5948f63502ae4, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/08ef6277e6a14d21b000ac155aeeff9e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/7afe6fc59f874e2993851f1ee3442e3f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/04b61c393151462387601e930870cc1d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ba2839bc3edb466f92422e3653da2da1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ea302e100b014838aa927da775388e96, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/6c0f33f805ba44a5805fa8b4650fe08e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/39c37f41356240e0be5f22966bea5ee8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/94ab2d948333445398893ac45f37a74d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/0257d6b6f1bf49b7a7eef7431836c7d2, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/4702dbd912a241e6bd64c03e80a0b1bb, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/79620ddb7924483680a90af17557566f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/d3dbdd7155184abaa0eea7c50ecfb872, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/58270f62e1894e019f5bd869533cd07f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/fcc802a3582d4e8d836568b5f3f6a8fc, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/bb2b4c37ea8649fab5b74425f7c62787, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/e71ce060e69241d092ac9a407c4773df, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/cf9913591cfc49a1b91e7c97685419ea] to archive 2024-11-11T03:42:13,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:42:13,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/82223b7cdf8947b6b04227fc77696299 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/82223b7cdf8947b6b04227fc77696299 2024-11-11T03:42:13,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/31e180ea5f234bc99ce8b29892d3b784 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/31e180ea5f234bc99ce8b29892d3b784 2024-11-11T03:42:13,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/81b0b1e036de40e09fd958b7c8d0fa8d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/81b0b1e036de40e09fd958b7c8d0fa8d 2024-11-11T03:42:13,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/c421be37e4b747dbb852f2bd8c387ecb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/c421be37e4b747dbb852f2bd8c387ecb 2024-11-11T03:42:13,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/93c2d21563a8480c9cf5948f63502ae4 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/93c2d21563a8480c9cf5948f63502ae4 2024-11-11T03:42:13,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/08ef6277e6a14d21b000ac155aeeff9e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/08ef6277e6a14d21b000ac155aeeff9e 2024-11-11T03:42:13,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/7afe6fc59f874e2993851f1ee3442e3f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/7afe6fc59f874e2993851f1ee3442e3f 2024-11-11T03:42:13,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/04b61c393151462387601e930870cc1d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/04b61c393151462387601e930870cc1d 2024-11-11T03:42:13,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ba2839bc3edb466f92422e3653da2da1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ba2839bc3edb466f92422e3653da2da1 2024-11-11T03:42:13,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ea302e100b014838aa927da775388e96 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/ea302e100b014838aa927da775388e96 2024-11-11T03:42:13,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/6c0f33f805ba44a5805fa8b4650fe08e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/6c0f33f805ba44a5805fa8b4650fe08e 2024-11-11T03:42:13,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/39c37f41356240e0be5f22966bea5ee8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/39c37f41356240e0be5f22966bea5ee8 2024-11-11T03:42:13,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/94ab2d948333445398893ac45f37a74d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/94ab2d948333445398893ac45f37a74d 2024-11-11T03:42:13,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/0257d6b6f1bf49b7a7eef7431836c7d2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/0257d6b6f1bf49b7a7eef7431836c7d2 2024-11-11T03:42:13,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/4702dbd912a241e6bd64c03e80a0b1bb to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/4702dbd912a241e6bd64c03e80a0b1bb 2024-11-11T03:42:13,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/79620ddb7924483680a90af17557566f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/79620ddb7924483680a90af17557566f 2024-11-11T03:42:13,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/d3dbdd7155184abaa0eea7c50ecfb872 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/d3dbdd7155184abaa0eea7c50ecfb872 2024-11-11T03:42:13,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/58270f62e1894e019f5bd869533cd07f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/58270f62e1894e019f5bd869533cd07f 2024-11-11T03:42:13,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/fcc802a3582d4e8d836568b5f3f6a8fc to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/fcc802a3582d4e8d836568b5f3f6a8fc 2024-11-11T03:42:13,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/bb2b4c37ea8649fab5b74425f7c62787 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/bb2b4c37ea8649fab5b74425f7c62787 2024-11-11T03:42:13,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/e71ce060e69241d092ac9a407c4773df to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/e71ce060e69241d092ac9a407c4773df 2024-11-11T03:42:13,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/cf9913591cfc49a1b91e7c97685419ea to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/cf9913591cfc49a1b91e7c97685419ea 2024-11-11T03:42:13,420 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/recovered.edits/473.seqid, newMaxSeqId=473, maxSeqId=1 2024-11-11T03:42:13,421 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. 2024-11-11T03:42:13,421 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for e07956485faf1a58beb177dd117c82c8: Waiting for close lock at 1731296531942Running coprocessor pre-close hooks at 1731296531942Disabling compacts and flushes for region at 1731296531942Disabling writes for close at 1731296532062 (+120 ms)Obtaining lock to block concurrent updates at 1731296532062Preparing flush snapshotting stores in e07956485faf1a58beb177dd117c82c8 at 1731296532062Finished memstore snapshotting TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731296532062Flushing stores of TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8. at 1731296532062Flushing e07956485faf1a58beb177dd117c82c8/A: creating writer at 1731296532062Flushing e07956485faf1a58beb177dd117c82c8/A: appending metadata at 1731296532064 (+2 ms)Flushing e07956485faf1a58beb177dd117c82c8/A: closing flushed file at 1731296532064Flushing e07956485faf1a58beb177dd117c82c8/B: creating writer at 1731296532476 (+412 ms)Flushing e07956485faf1a58beb177dd117c82c8/B: appending metadata at 1731296532481 (+5 ms)Flushing e07956485faf1a58beb177dd117c82c8/B: closing flushed file at 1731296532481Flushing e07956485faf1a58beb177dd117c82c8/C: creating writer at 1731296532896 (+415 ms)Flushing e07956485faf1a58beb177dd117c82c8/C: appending metadata at 1731296532900 (+4 ms)Flushing e07956485faf1a58beb177dd117c82c8/C: closing flushed file at 1731296532900Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10e3ba6f: reopening flushed file at 1731296533316 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25af1099: reopening flushed file at 1731296533320 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c519a65: reopening flushed file at 1731296533325 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e07956485faf1a58beb177dd117c82c8 in 1267ms, sequenceid=470, compaction requested=true at 1731296533330 (+5 ms)Writing region close event to WAL at 1731296533418 (+88 ms)Running coprocessor post-close hooks at 1731296533421 (+3 ms)Closed at 1731296533421 2024-11-11T03:42:13,423 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:13,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e07956485faf1a58beb177dd117c82c8, regionState=CLOSED 2024-11-11T03:42:13,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:42:13,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T03:42:13,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure e07956485faf1a58beb177dd117c82c8, server=3ffe7f9d089e,34473,1731296384288 in 1.6360 sec 2024-11-11T03:42:13,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-11T03:42:13,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e07956485faf1a58beb177dd117c82c8, UNASSIGN in 1.6410 sec 2024-11-11T03:42:13,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T03:42:13,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6440 sec 2024-11-11T03:42:13,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296533432"}]},"ts":"1731296533432"} 2024-11-11T03:42:13,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T03:42:13,433 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T03:42:13,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6560 sec 2024-11-11T03:42:13,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T03:42:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T03:42:13,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:13,935 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T03:42:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:13,940 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T03:42:13,942 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:13,944 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:13,946 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/recovered.edits] 2024-11-11T03:42:13,948 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2f7d03785d8f429898fb6a5c3339df78 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/2f7d03785d8f429898fb6a5c3339df78 2024-11-11T03:42:13,949 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/41cdcd63f9524a8ea670c86b9d8271bf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/41cdcd63f9524a8ea670c86b9d8271bf 2024-11-11T03:42:13,949 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71f7e54bc4c84fcea0bb9487a61c2c08 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/71f7e54bc4c84fcea0bb9487a61c2c08 2024-11-11T03:42:13,950 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7e50919d25884e7187b734865f5e914f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/A/7e50919d25884e7187b734865f5e914f 2024-11-11T03:42:13,952 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8769276ddb224f379ad509a1f46e3ec2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/8769276ddb224f379ad509a1f46e3ec2 2024-11-11T03:42:13,953 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba44024d68cf4e0e8523da70e5ec0880 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/ba44024d68cf4e0e8523da70e5ec0880 2024-11-11T03:42:13,954 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/e5f1d087d43e4c94a64cf968a83f8a08 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/e5f1d087d43e4c94a64cf968a83f8a08 2024-11-11T03:42:13,955 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f6e35206f62148babbe7b6b2bdce5aff to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/B/f6e35206f62148babbe7b6b2bdce5aff 2024-11-11T03:42:13,958 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/1525ff4a00244b729ef0545122c425ea to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/1525ff4a00244b729ef0545122c425ea 2024-11-11T03:42:13,959 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/56a19f8c82474809815c63ab7fc8a125 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/56a19f8c82474809815c63ab7fc8a125 2024-11-11T03:42:13,960 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/9a3db68215394a7bb1e7a75772a04159 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/9a3db68215394a7bb1e7a75772a04159 2024-11-11T03:42:13,962 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/efdf84aa9ecc40729ac890930731830a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/C/efdf84aa9ecc40729ac890930731830a 2024-11-11T03:42:13,965 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/recovered.edits/473.seqid to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8/recovered.edits/473.seqid 2024-11-11T03:42:13,965 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/e07956485faf1a58beb177dd117c82c8 2024-11-11T03:42:13,965 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T03:42:13,968 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:13,971 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T03:42:13,974 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T03:42:13,975 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:13,975 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T03:42:13,976 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731296533975"}]},"ts":"9223372036854775807"} 2024-11-11T03:42:13,978 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T03:42:13,978 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e07956485faf1a58beb177dd117c82c8, NAME => 'TestAcidGuarantees,,1731296508329.e07956485faf1a58beb177dd117c82c8.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T03:42:13,978 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T03:42:13,978 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731296533978"}]},"ts":"9223372036854775807"} 2024-11-11T03:42:13,980 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T03:42:13,980 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:13,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-11T03:42:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T03:42:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T03:42:14,052 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:14,065 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 252), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 295) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11797 (was 11808) 2024-11-11T03:42:14,077 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=11797 2024-11-11T03:42:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:42:14,079 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:42:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:14,081 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T03:42:14,081 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:14,081 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-11T03:42:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T03:42:14,082 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T03:42:14,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742331_1507 (size=1061) 2024-11-11T03:42:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T03:42:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T03:42:14,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c 2024-11-11T03:42:14,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742332_1508 (size=53) 2024-11-11T03:42:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T03:42:14,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:42:14,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d3a8571217c3f763e5d9d9614d29758b, disabling compactions & flushes 2024-11-11T03:42:14,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:14,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:14,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. after waiting 0 ms 2024-11-11T03:42:14,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:14,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:14,947 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3a8571217c3f763e5d9d9614d29758b: Waiting for close lock at 1731296534946Disabling compacts and flushes for region at 1731296534946Disabling writes for close at 1731296534946Writing region close event to WAL at 1731296534946Closed at 1731296534946 2024-11-11T03:42:14,949 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T03:42:14,950 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731296534949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731296534949"}]},"ts":"1731296534949"} 2024-11-11T03:42:14,954 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T03:42:14,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T03:42:14,956 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296534956"}]},"ts":"1731296534956"} 2024-11-11T03:42:14,959 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T03:42:14,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, ASSIGN}] 2024-11-11T03:42:14,961 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, ASSIGN 2024-11-11T03:42:14,961 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, ASSIGN; state=OFFLINE, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=false 2024-11-11T03:42:15,113 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:15,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, ASSIGN because future has completed 2024-11-11T03:42:15,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:42:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T03:42:15,277 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:15,277 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:42:15,277 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,277 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:42:15,277 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,277 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,279 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,280 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:42:15,280 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3a8571217c3f763e5d9d9614d29758b columnFamilyName A 2024-11-11T03:42:15,280 DEBUG [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:15,281 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(327): Store=d3a8571217c3f763e5d9d9614d29758b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:42:15,281 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,282 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:42:15,282 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3a8571217c3f763e5d9d9614d29758b columnFamilyName B 2024-11-11T03:42:15,282 DEBUG [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:15,282 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(327): Store=d3a8571217c3f763e5d9d9614d29758b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:42:15,282 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,283 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:42:15,283 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3a8571217c3f763e5d9d9614d29758b columnFamilyName C 2024-11-11T03:42:15,283 DEBUG [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:15,284 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(327): Store=d3a8571217c3f763e5d9d9614d29758b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:42:15,284 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:15,284 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,284 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,285 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,285 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,285 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,286 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:42:15,287 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,288 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:42:15,288 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened d3a8571217c3f763e5d9d9614d29758b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70322963, jitterRate=0.04789380729198456}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:42:15,288 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:15,289 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for d3a8571217c3f763e5d9d9614d29758b: Running coprocessor pre-open hook at 1731296535277Writing region info on filesystem at 1731296535277Initializing all the Stores at 1731296535278 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296535278Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296535278Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296535278Cleaning up temporary data from old regions at 1731296535285 (+7 ms)Running coprocessor post-open hooks at 1731296535288 (+3 ms)Region opened successfully at 1731296535289 (+1 ms) 2024-11-11T03:42:15,290 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., pid=146, masterSystemTime=1731296535272 2024-11-11T03:42:15,291 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:15,291 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:15,291 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=OPEN, openSeqNum=2, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:15,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:42:15,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-11T03:42:15,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 in 176 msec 2024-11-11T03:42:15,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-11T03:42:15,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, ASSIGN in 335 msec 2024-11-11T03:42:15,296 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T03:42:15,296 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296535296"}]},"ts":"1731296535296"} 2024-11-11T03:42:15,297 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T03:42:15,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T03:42:15,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2190 sec 2024-11-11T03:42:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T03:42:16,233 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:16,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:16,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:16,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:16,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444cdad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:16,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:16,241 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e76c8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:16,243 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:16,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:16,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:16,245 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T03:42:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ffe7f9d089e,45295,1731296383499 2024-11-11T03:42:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d3b5fd6 2024-11-11T03:42:16,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:42:16,246 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:42:16,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T03:42:16,248 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:42:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:16,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742333_1509 (size=1097) 2024-11-11T03:42:16,661 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T03:42:16,661 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T03:42:16,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:42:16,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, REOPEN/MOVE}] 2024-11-11T03:42:16,669 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, REOPEN/MOVE 2024-11-11T03:42:16,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:16,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, REOPEN/MOVE because future has completed 2024-11-11T03:42:16,672 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:42:16,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:42:16,826 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:16,826 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:42:16,826 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing d3a8571217c3f763e5d9d9614d29758b, disabling compactions & flushes 2024-11-11T03:42:16,826 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:16,827 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:16,827 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. after waiting 0 ms 2024-11-11T03:42:16,827 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:16,835 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T03:42:16,835 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:16,836 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for d3a8571217c3f763e5d9d9614d29758b: Waiting for close lock at 1731296536826Running coprocessor pre-close hooks at 1731296536826Disabling compacts and flushes for region at 1731296536826Disabling writes for close at 1731296536827 (+1 ms)Writing region close event to WAL at 1731296536828 (+1 ms)Running coprocessor post-close hooks at 1731296536835 (+7 ms)Closed at 1731296536835 2024-11-11T03:42:16,836 WARN [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: d3a8571217c3f763e5d9d9614d29758b to self. 2024-11-11T03:42:16,838 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:16,839 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=CLOSED 2024-11-11T03:42:16,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:42:16,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-11T03:42:16,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 in 170 msec 2024-11-11T03:42:16,846 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, REOPEN/MOVE; state=CLOSED, location=3ffe7f9d089e,34473,1731296384288; forceNewPlan=false, retain=true 2024-11-11T03:42:16,996 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=OPENING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:16,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, REOPEN/MOVE because future has completed 2024-11-11T03:42:16,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:42:17,156 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,157 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:42:17,157 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,157 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:42:17,158 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,158 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,160 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,161 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:42:17,162 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3a8571217c3f763e5d9d9614d29758b columnFamilyName A 2024-11-11T03:42:17,163 DEBUG [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:17,163 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(327): Store=d3a8571217c3f763e5d9d9614d29758b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:42:17,164 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,164 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:42:17,164 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3a8571217c3f763e5d9d9614d29758b columnFamilyName B 2024-11-11T03:42:17,164 DEBUG [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:17,165 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(327): Store=d3a8571217c3f763e5d9d9614d29758b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:42:17,165 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,166 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T03:42:17,166 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3a8571217c3f763e5d9d9614d29758b columnFamilyName C 2024-11-11T03:42:17,166 DEBUG [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:17,166 INFO [StoreOpener-d3a8571217c3f763e5d9d9614d29758b-1 {}] regionserver.HStore(327): Store=d3a8571217c3f763e5d9d9614d29758b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:42:17,166 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,166 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,167 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,168 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,169 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,169 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,170 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T03:42:17,171 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,172 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened d3a8571217c3f763e5d9d9614d29758b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68563235, jitterRate=0.02167181670665741}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T03:42:17,172 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,173 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for d3a8571217c3f763e5d9d9614d29758b: Running coprocessor pre-open hook at 1731296537158Writing region info on filesystem at 1731296537158Initializing all the Stores at 1731296537159 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296537159Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296537160 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731296537160Cleaning up temporary data from old regions at 1731296537169 (+9 ms)Running coprocessor post-open hooks at 1731296537172 (+3 ms)Region opened successfully at 1731296537173 (+1 ms) 2024-11-11T03:42:17,174 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., pid=151, masterSystemTime=1731296537150 2024-11-11T03:42:17,175 DEBUG [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,175 INFO [RS_OPEN_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,176 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=OPEN, openSeqNum=5, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:42:17,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-11T03:42:17,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 in 180 msec 2024-11-11T03:42:17,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T03:42:17,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, REOPEN/MOVE in 511 msec 2024-11-11T03:42:17,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-11T03:42:17,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-11T03:42:17,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 935 msec 2024-11-11T03:42:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T03:42:17,185 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:17,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,186 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138ee482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,187 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@165d550f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,189 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a19e166, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,189 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4c816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,191 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72958341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f8e124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,193 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57387bc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c310f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,196 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f910db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,197 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9dbae09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,199 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddbdd95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47203320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,202 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d88fbcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,203 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6bb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b53517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@954c5d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,209 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ee47e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,210 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b2ee2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ffe7f9d089e,45295,-1 for getting cluster id 2024-11-11T03:42:17,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T03:42:17,212 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f38a4edf-cf51-45c3-8d97-e3140b469b36' 2024-11-11T03:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T03:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f38a4edf-cf51-45c3-8d97-e3140b469b36" 2024-11-11T03:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f7204e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ffe7f9d089e,45295,-1] 2024-11-11T03:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T03:42:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:17,213 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T03:42:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27969691, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:42:17,214 DEBUG [Thread-2281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,214 DEBUG [Thread-2298 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,214 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:17,214 DEBUG [Thread-2296 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,215 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,215 DEBUG [Thread-2279 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,215 DEBUG [Thread-2290 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,215 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,216 DEBUG [Thread-2294 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T03:42:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,216 DEBUG [Thread-2294 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,217 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,217 DEBUG [Thread-2294 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,217 DEBUG [Thread-2292 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T03:42:17,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:17,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,223 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T03:42:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ffe7f9d089e,34473,1731296384288, seqNum=-1] 2024-11-11T03:42:17,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:42:17,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:42:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5] 2024-11-11T03:42:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T03:42:17,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:17,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:17,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:17,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:17,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:17,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:17,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cf805a018d5c4591a6caa2565a371d2b_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296537228/Put/seqid=0 2024-11-11T03:42:17,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296537240, max timestamp=1731296537258Num uniques 859; 2024-11-11T03:42:17,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:17,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296537240, max timestamp=1731296537258Num uniques 859; 2024-11-11T03:42:17,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731296537240, max timestamp=1731296537258Num uniques 859; 2024-11-11T03:42:17,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:17,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T03:42:17,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742334_1510 (size=9714) 2024-11-11T03:42:17,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:17,273 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cf805a018d5c4591a6caa2565a371d2b_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf805a018d5c4591a6caa2565a371d2b_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,274 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/2721487713ed4739bd3999e633276301, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:17,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/2721487713ed4739bd3999e633276301 is 175, key is test_row_0/A:col10/1731296537228/Put/seqid=0 2024-11-11T03:42:17,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742335_1511 (size=22361) 2024-11-11T03:42:17,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/2721487713ed4739bd3999e633276301 2024-11-11T03:42:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296597278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296597280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296597281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/3ea30350754c4b7f8f079bbe463b1d54 is 50, key is test_row_0/B:col10/1731296537228/Put/seqid=0 2024-11-11T03:42:17,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742336_1512 (size=9657) 2024-11-11T03:42:17,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/3ea30350754c4b7f8f079bbe463b1d54 2024-11-11T03:42:17,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731296537274, max timestamp=1731296537280Num uniques 1018; 2024-11-11T03:42:17,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T03:42:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731296537274, max timestamp=1731296537280Num uniques 1018; 2024-11-11T03:42:17,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-11T03:42:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731296537274, max timestamp=1731296537280Num uniques 1018; 2024-11-11T03:42:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3ffe7f9d089e:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-11T03:42:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T03:42:17,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/5bfeef99a6c7431ba560a606268dfb05 is 50, key is test_row_0/C:col10/1731296537228/Put/seqid=0 2024-11-11T03:42:17,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742337_1513 (size=9657) 2024-11-11T03:42:17,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296597350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296597350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T03:42:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296597401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296597402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296597402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296597461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296597461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T03:42:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T03:42:17,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296597612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296597612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296597612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296597671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296597671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T03:42:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:17,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/5bfeef99a6c7431ba560a606268dfb05 2024-11-11T03:42:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/2721487713ed4739bd3999e633276301 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/2721487713ed4739bd3999e633276301 2024-11-11T03:42:17,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/2721487713ed4739bd3999e633276301, entries=100, sequenceid=15, filesize=21.8 K 2024-11-11T03:42:17,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/3ea30350754c4b7f8f079bbe463b1d54 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3ea30350754c4b7f8f079bbe463b1d54 2024-11-11T03:42:17,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3ea30350754c4b7f8f079bbe463b1d54, entries=100, sequenceid=15, filesize=9.4 K 2024-11-11T03:42:17,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/5bfeef99a6c7431ba560a606268dfb05 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/5bfeef99a6c7431ba560a606268dfb05 2024-11-11T03:42:17,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/5bfeef99a6c7431ba560a606268dfb05, entries=100, sequenceid=15, filesize=9.4 K 2024-11-11T03:42:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T03:42:17,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=128.11 KB/131184 for d3a8571217c3f763e5d9d9614d29758b in 528ms, sequenceid=15, compaction requested=false 2024-11-11T03:42:17,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:17,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T03:42:17,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:17,833 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=128.11 KB heapSize=365.64 KB 2024-11-11T03:42:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:17,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111190400ac047de489aac0afadc671ac084_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296537349/Put/seqid=0 2024-11-11T03:42:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T03:42:17,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742338_1514 (size=12154) 2024-11-11T03:42:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:17,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296597930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296597934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296597937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296597981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296597981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296598042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296598042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296598042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:18,250 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111190400ac047de489aac0afadc671ac084_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111190400ac047de489aac0afadc671ac084_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:18,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/41171f81afd94b8390a718d0fa19bc1a, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/41171f81afd94b8390a718d0fa19bc1a is 175, key is test_row_0/A:col10/1731296537349/Put/seqid=0 2024-11-11T03:42:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296598252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296598252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296598252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742339_1515 (size=30955) 2024-11-11T03:42:18,260 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/41171f81afd94b8390a718d0fa19bc1a 2024-11-11T03:42:18,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/6b32a2e866094aad8a786fea59fc3150 is 50, key is test_row_0/B:col10/1731296537349/Put/seqid=0 2024-11-11T03:42:18,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742340_1516 (size=12001) 2024-11-11T03:42:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T03:42:18,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296598492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296598492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296598562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296598562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296598562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:18,672 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/6b32a2e866094aad8a786fea59fc3150 2024-11-11T03:42:18,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/d28bf357413d4303a874332aea4dde9a is 50, key is test_row_0/C:col10/1731296537349/Put/seqid=0 2024-11-11T03:42:18,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742341_1517 (size=12001) 2024-11-11T03:42:18,937 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:42:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296599072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296599072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296599072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/d28bf357413d4303a874332aea4dde9a 2024-11-11T03:42:19,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/41171f81afd94b8390a718d0fa19bc1a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/41171f81afd94b8390a718d0fa19bc1a 2024-11-11T03:42:19,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/41171f81afd94b8390a718d0fa19bc1a, entries=150, sequenceid=68, filesize=30.2 K 2024-11-11T03:42:19,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/6b32a2e866094aad8a786fea59fc3150 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6b32a2e866094aad8a786fea59fc3150 2024-11-11T03:42:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6b32a2e866094aad8a786fea59fc3150, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T03:42:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/d28bf357413d4303a874332aea4dde9a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/d28bf357413d4303a874332aea4dde9a 2024-11-11T03:42:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,102 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/d28bf357413d4303a874332aea4dde9a, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T03:42:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~128.11 KB/131184, heapSize ~365.59 KB/374368, currentSize=53.67 KB/54960 for d3a8571217c3f763e5d9d9614d29758b in 1270ms, sequenceid=68, compaction requested=false 2024-11-11T03:42:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T03:42:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T03:42:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T03:42:19,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-11T03:42:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8920 sec 2024-11-11T03:42:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T03:42:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,362 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,363 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T03:42:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T03:42:19,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,366 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:19,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T03:42:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:19,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T03:42:19,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:19,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:19,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:19,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:19,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T03:42:19,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:19,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:19,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:19,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296599542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296599546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e753f05d49b94c33944da824f067942d_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296539503/Put/seqid=0 2024-11-11T03:42:19,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742342_1518 (size=14594) 2024-11-11T03:42:19,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296599652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296599652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T03:42:19,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T03:42:19,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:19,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:19,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:19,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T03:42:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:19,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:19,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296599862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296599862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:19,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:19,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e753f05d49b94c33944da824f067942d_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e753f05d49b94c33944da824f067942d_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:19,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/42971d894cf8472780eace9c5b6bd89a, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:19,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/42971d894cf8472780eace9c5b6bd89a is 175, key is test_row_0/A:col10/1731296539503/Put/seqid=0 2024-11-11T03:42:19,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742343_1519 (size=39549) 2024-11-11T03:42:19,965 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/42971d894cf8472780eace9c5b6bd89a 2024-11-11T03:42:19,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/1cb490baf51647eeaaacb0e9b14dee13 is 50, key is test_row_0/B:col10/1731296539503/Put/seqid=0 2024-11-11T03:42:19,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742344_1520 (size=12001) 2024-11-11T03:42:19,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/1cb490baf51647eeaaacb0e9b14dee13 2024-11-11T03:42:19,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b7f34dacffc04b13bfadb286f874cf34 is 50, key is test_row_0/C:col10/1731296539503/Put/seqid=0 2024-11-11T03:42:19,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742345_1521 (size=12001) 2024-11-11T03:42:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T03:42:20,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T03:42:20,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:20,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296600081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,085 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296600092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296600092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,095 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,095 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T03:42:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296600171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296600172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T03:42:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:20,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b7f34dacffc04b13bfadb286f874cf34 2024-11-11T03:42:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/42971d894cf8472780eace9c5b6bd89a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/42971d894cf8472780eace9c5b6bd89a 2024-11-11T03:42:20,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/42971d894cf8472780eace9c5b6bd89a, entries=200, sequenceid=81, filesize=38.6 K 2024-11-11T03:42:20,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/1cb490baf51647eeaaacb0e9b14dee13 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/1cb490baf51647eeaaacb0e9b14dee13 2024-11-11T03:42:20,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/1cb490baf51647eeaaacb0e9b14dee13, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T03:42:20,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b7f34dacffc04b13bfadb286f874cf34 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b7f34dacffc04b13bfadb286f874cf34 2024-11-11T03:42:20,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b7f34dacffc04b13bfadb286f874cf34, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T03:42:20,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d3a8571217c3f763e5d9d9614d29758b in 891ms, sequenceid=81, compaction requested=true 2024-11-11T03:42:20,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:20,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:20,396 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:20,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:20,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:20,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:20,396 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:20,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:20,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/A is initiating minor compaction (all files) 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/B is initiating minor compaction (all files) 2024-11-11T03:42:20,397 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/A in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,397 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/B in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,397 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3ea30350754c4b7f8f079bbe463b1d54, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6b32a2e866094aad8a786fea59fc3150, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/1cb490baf51647eeaaacb0e9b14dee13] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=32.9 K 2024-11-11T03:42:20,397 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/2721487713ed4739bd3999e633276301, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/41171f81afd94b8390a718d0fa19bc1a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/42971d894cf8472780eace9c5b6bd89a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=90.7 K 2024-11-11T03:42:20,397 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/2721487713ed4739bd3999e633276301, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/41171f81afd94b8390a718d0fa19bc1a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/42971d894cf8472780eace9c5b6bd89a] 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ea30350754c4b7f8f079bbe463b1d54, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731296537228 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2721487713ed4739bd3999e633276301, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731296537228 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b32a2e866094aad8a786fea59fc3150, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731296537333 2024-11-11T03:42:20,397 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41171f81afd94b8390a718d0fa19bc1a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731296537333 2024-11-11T03:42:20,398 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb490baf51647eeaaacb0e9b14dee13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296537925 2024-11-11T03:42:20,398 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42971d894cf8472780eace9c5b6bd89a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296537925 2024-11-11T03:42:20,408 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:20,410 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:20,410 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/74d2bf713d8a4615a733f86a5b75d16d is 50, key is test_row_0/B:col10/1731296539503/Put/seqid=0 2024-11-11T03:42:20,414 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e1c6aea261804516b3452a837aef50af_d3a8571217c3f763e5d9d9614d29758b store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:20,416 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e1c6aea261804516b3452a837aef50af_d3a8571217c3f763e5d9d9614d29758b, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:20,416 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e1c6aea261804516b3452a837aef50af_d3a8571217c3f763e5d9d9614d29758b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:20,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742346_1522 (size=12104) 2024-11-11T03:42:20,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742347_1523 (size=4469) 2024-11-11T03:42:20,430 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#A#compaction#444 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:20,431 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/0818b9b105b74bce83bd76c95689144c is 175, key is test_row_0/A:col10/1731296539503/Put/seqid=0 2024-11-11T03:42:20,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742348_1524 (size=31058) 2024-11-11T03:42:20,442 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/0818b9b105b74bce83bd76c95689144c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/0818b9b105b74bce83bd76c95689144c 2024-11-11T03:42:20,446 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/A of d3a8571217c3f763e5d9d9614d29758b into 0818b9b105b74bce83bd76c95689144c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:20,446 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:20,446 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/A, priority=13, startTime=1731296540396; duration=0sec 2024-11-11T03:42:20,446 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:20,446 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:A 2024-11-11T03:42:20,446 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:20,447 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:20,448 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/C is initiating minor compaction (all files) 2024-11-11T03:42:20,448 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/C in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,448 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/5bfeef99a6c7431ba560a606268dfb05, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/d28bf357413d4303a874332aea4dde9a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b7f34dacffc04b13bfadb286f874cf34] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=32.9 K 2024-11-11T03:42:20,449 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bfeef99a6c7431ba560a606268dfb05, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731296537228 2024-11-11T03:42:20,449 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28bf357413d4303a874332aea4dde9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731296537333 2024-11-11T03:42:20,449 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f34dacffc04b13bfadb286f874cf34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296537925 2024-11-11T03:42:20,461 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#C#compaction#445 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:20,461 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0a5c8aa0a69a4505a2365a89af82f6d0 is 50, key is test_row_0/C:col10/1731296539503/Put/seqid=0 2024-11-11T03:42:20,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T03:42:20,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:20,464 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T03:42:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742349_1525 (size=12104) 2024-11-11T03:42:20,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:20,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:20,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:20,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:20,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:20,469 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0a5c8aa0a69a4505a2365a89af82f6d0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a5c8aa0a69a4505a2365a89af82f6d0 2024-11-11T03:42:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111124a58db8c26e4895b4f06c0c0a8b4a69_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296539545/Put/seqid=0 2024-11-11T03:42:20,476 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/C of d3a8571217c3f763e5d9d9614d29758b into 0a5c8aa0a69a4505a2365a89af82f6d0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:20,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:20,476 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/C, priority=13, startTime=1731296540396; duration=0sec 2024-11-11T03:42:20,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:20,476 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:C 2024-11-11T03:42:20,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742350_1526 (size=12154) 2024-11-11T03:42:20,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:20,487 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111124a58db8c26e4895b4f06c0c0a8b4a69_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124a58db8c26e4895b4f06c0c0a8b4a69_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/d5cfe9ff454f450394ff6020b36bd4c9, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/d5cfe9ff454f450394ff6020b36bd4c9 is 175, key is test_row_0/A:col10/1731296539545/Put/seqid=0 2024-11-11T03:42:20,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742351_1527 (size=30955) 2024-11-11T03:42:20,492 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/d5cfe9ff454f450394ff6020b36bd4c9 2024-11-11T03:42:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/b8437efbb6c94d72882dd28bd9b1e5b0 is 50, key is test_row_0/B:col10/1731296539545/Put/seqid=0 2024-11-11T03:42:20,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742352_1528 (size=12001) 2024-11-11T03:42:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T03:42:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:20,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296600705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296600706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296600812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,822 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/74d2bf713d8a4615a733f86a5b75d16d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/74d2bf713d8a4615a733f86a5b75d16d 2024-11-11T03:42:20,825 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/B of d3a8571217c3f763e5d9d9614d29758b into 74d2bf713d8a4615a733f86a5b75d16d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:20,825 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:20,825 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/B, priority=13, startTime=1731296540396; duration=0sec 2024-11-11T03:42:20,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:20,825 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:20,825 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:B 2024-11-11T03:42:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296600822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:20,901 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/b8437efbb6c94d72882dd28bd9b1e5b0 2024-11-11T03:42:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/3ca7c9ed11c74aebbb52c8fa1d940953 is 50, key is test_row_0/C:col10/1731296539545/Put/seqid=0 2024-11-11T03:42:20,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742353_1529 (size=12001) 2024-11-11T03:42:21,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296601021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296601032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,310 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/3ca7c9ed11c74aebbb52c8fa1d940953 2024-11-11T03:42:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/d5cfe9ff454f450394ff6020b36bd4c9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/d5cfe9ff454f450394ff6020b36bd4c9 2024-11-11T03:42:21,315 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/d5cfe9ff454f450394ff6020b36bd4c9, entries=150, sequenceid=101, filesize=30.2 K 2024-11-11T03:42:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/b8437efbb6c94d72882dd28bd9b1e5b0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/b8437efbb6c94d72882dd28bd9b1e5b0 2024-11-11T03:42:21,318 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/b8437efbb6c94d72882dd28bd9b1e5b0, entries=150, sequenceid=101, filesize=11.7 K 2024-11-11T03:42:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/3ca7c9ed11c74aebbb52c8fa1d940953 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/3ca7c9ed11c74aebbb52c8fa1d940953 2024-11-11T03:42:21,321 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/3ca7c9ed11c74aebbb52c8fa1d940953, entries=150, sequenceid=101, filesize=11.7 K 2024-11-11T03:42:21,321 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d3a8571217c3f763e5d9d9614d29758b in 857ms, sequenceid=101, compaction requested=false 2024-11-11T03:42:21,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:21,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:21,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T03:42:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T03:42:21,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T03:42:21,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9570 sec 2024-11-11T03:42:21,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.9620 sec 2024-11-11T03:42:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:21,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T03:42:21,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:21,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:21,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:21,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:21,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:21,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:21,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111182dc8ecc5a0b4054abbf13c2efba56be_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296540704/Put/seqid=0 2024-11-11T03:42:21,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742354_1530 (size=14594) 2024-11-11T03:42:21,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296601374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296601376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296601482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296601491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T03:42:21,512 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:21,512 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-11T03:42:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T03:42:21,514 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:21,514 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:21,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T03:42:21,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T03:42:21,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:21,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:21,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:21,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296601692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296601701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:21,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:21,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111182dc8ecc5a0b4054abbf13c2efba56be_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111182dc8ecc5a0b4054abbf13c2efba56be_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:21,745 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/bbdc543a9c5747a7b2fa672348dfc096, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:21,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/bbdc543a9c5747a7b2fa672348dfc096 is 175, key is test_row_0/A:col10/1731296540704/Put/seqid=0 2024-11-11T03:42:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742355_1531 (size=39549) 2024-11-11T03:42:21,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T03:42:21,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:21,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:21,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:21,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T03:42:21,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T03:42:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:21,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:22,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296602002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296602012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296602091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,094 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296602112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,115 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T03:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:22,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:22,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296602122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,126 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:22,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T03:42:22,149 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/bbdc543a9c5747a7b2fa672348dfc096 2024-11-11T03:42:22,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/e183766c0fe745fa9975eb6d466e74e8 is 50, key is test_row_0/B:col10/1731296540704/Put/seqid=0 2024-11-11T03:42:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742356_1532 (size=12001) 2024-11-11T03:42:22,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/e183766c0fe745fa9975eb6d466e74e8 2024-11-11T03:42:22,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/ef77206a1b3e4b3c99100360d4ff6329 is 50, key is test_row_0/C:col10/1731296540704/Put/seqid=0 2024-11-11T03:42:22,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742357_1533 (size=12001) 2024-11-11T03:42:22,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/ef77206a1b3e4b3c99100360d4ff6329 2024-11-11T03:42:22,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/bbdc543a9c5747a7b2fa672348dfc096 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/bbdc543a9c5747a7b2fa672348dfc096 2024-11-11T03:42:22,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/bbdc543a9c5747a7b2fa672348dfc096, entries=200, sequenceid=117, filesize=38.6 K 2024-11-11T03:42:22,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/e183766c0fe745fa9975eb6d466e74e8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e183766c0fe745fa9975eb6d466e74e8 2024-11-11T03:42:22,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e183766c0fe745fa9975eb6d466e74e8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T03:42:22,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/ef77206a1b3e4b3c99100360d4ff6329 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ef77206a1b3e4b3c99100360d4ff6329 2024-11-11T03:42:22,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ef77206a1b3e4b3c99100360d4ff6329, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T03:42:22,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d3a8571217c3f763e5d9d9614d29758b in 844ms, sequenceid=117, compaction requested=true 2024-11-11T03:42:22,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:22,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:22,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:22,178 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:22,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:22,178 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:22,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:22,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:22,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:22,179 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:22,179 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:22,179 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/A is initiating minor compaction (all files) 2024-11-11T03:42:22,179 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/B is initiating minor compaction (all files) 2024-11-11T03:42:22,179 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/A in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:22,179 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/B in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:22,179 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/0818b9b105b74bce83bd76c95689144c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/d5cfe9ff454f450394ff6020b36bd4c9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/bbdc543a9c5747a7b2fa672348dfc096] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=99.2 K 2024-11-11T03:42:22,179 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/74d2bf713d8a4615a733f86a5b75d16d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/b8437efbb6c94d72882dd28bd9b1e5b0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e183766c0fe745fa9975eb6d466e74e8] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=35.3 K 2024-11-11T03:42:22,179 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:22,179 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/0818b9b105b74bce83bd76c95689144c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/d5cfe9ff454f450394ff6020b36bd4c9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/bbdc543a9c5747a7b2fa672348dfc096] 2024-11-11T03:42:22,180 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d2bf713d8a4615a733f86a5b75d16d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296537925 2024-11-11T03:42:22,180 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b8437efbb6c94d72882dd28bd9b1e5b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731296539536 2024-11-11T03:42:22,180 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0818b9b105b74bce83bd76c95689144c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296537925 2024-11-11T03:42:22,180 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e183766c0fe745fa9975eb6d466e74e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296540699 2024-11-11T03:42:22,180 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5cfe9ff454f450394ff6020b36bd4c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731296539536 2024-11-11T03:42:22,180 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbdc543a9c5747a7b2fa672348dfc096, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296540699 2024-11-11T03:42:22,185 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#B#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:22,185 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/6a81c306f0dd4db995adb8f30136bec1 is 50, key is test_row_0/B:col10/1731296540704/Put/seqid=0 2024-11-11T03:42:22,186 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:22,188 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111776b4758e9e9487f87c4513c6511543d_d3a8571217c3f763e5d9d9614d29758b store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:22,190 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111776b4758e9e9487f87c4513c6511543d_d3a8571217c3f763e5d9d9614d29758b, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:22,190 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111776b4758e9e9487f87c4513c6511543d_d3a8571217c3f763e5d9d9614d29758b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:22,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742358_1534 (size=12207) 2024-11-11T03:42:22,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742359_1535 (size=4469) 2024-11-11T03:42:22,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T03:42:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:22,275 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T03:42:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:22,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111126d5ebbd85a449df99224c0a4799ffbb_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296541374/Put/seqid=0 2024-11-11T03:42:22,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742360_1536 (size=12304) 2024-11-11T03:42:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:22,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:22,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296602542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296602543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,594 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#A#compaction#453 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:22,594 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/e8c55d006f7d4a2980b9beb9c3a9ed59 is 175, key is test_row_0/A:col10/1731296540704/Put/seqid=0 2024-11-11T03:42:22,597 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/6a81c306f0dd4db995adb8f30136bec1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6a81c306f0dd4db995adb8f30136bec1 2024-11-11T03:42:22,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742361_1537 (size=31161) 2024-11-11T03:42:22,602 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/B of d3a8571217c3f763e5d9d9614d29758b into 6a81c306f0dd4db995adb8f30136bec1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:22,602 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:22,602 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/B, priority=13, startTime=1731296542178; duration=0sec 2024-11-11T03:42:22,602 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:22,602 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:B 2024-11-11T03:42:22,602 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:22,602 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/e8c55d006f7d4a2980b9beb9c3a9ed59 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/e8c55d006f7d4a2980b9beb9c3a9ed59 2024-11-11T03:42:22,603 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:22,603 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/C is initiating minor compaction (all files) 2024-11-11T03:42:22,603 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/C in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:22,603 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a5c8aa0a69a4505a2365a89af82f6d0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/3ca7c9ed11c74aebbb52c8fa1d940953, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ef77206a1b3e4b3c99100360d4ff6329] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=35.3 K 2024-11-11T03:42:22,603 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a5c8aa0a69a4505a2365a89af82f6d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731296537925 2024-11-11T03:42:22,604 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca7c9ed11c74aebbb52c8fa1d940953, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731296539536 2024-11-11T03:42:22,604 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ef77206a1b3e4b3c99100360d4ff6329, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296540699 2024-11-11T03:42:22,606 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/A of d3a8571217c3f763e5d9d9614d29758b into e8c55d006f7d4a2980b9beb9c3a9ed59(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:22,606 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:22,606 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/A, priority=13, startTime=1731296542178; duration=0sec 2024-11-11T03:42:22,606 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:22,606 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:A 2024-11-11T03:42:22,610 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#C#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:22,610 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/ffeb90c21ac44132888c34aaf029ea7d is 50, key is test_row_0/C:col10/1731296540704/Put/seqid=0 2024-11-11T03:42:22,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742362_1538 (size=12207) 2024-11-11T03:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T03:42:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296602652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296602652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:22,686 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111126d5ebbd85a449df99224c0a4799ffbb_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111126d5ebbd85a449df99224c0a4799ffbb_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/35956794e04f49d297be9e1b8321676a, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:22,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/35956794e04f49d297be9e1b8321676a is 175, key is test_row_0/A:col10/1731296541374/Put/seqid=0 2024-11-11T03:42:22,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742363_1539 (size=31105) 2024-11-11T03:42:22,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296602862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296602862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:23,017 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/ffeb90c21ac44132888c34aaf029ea7d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ffeb90c21ac44132888c34aaf029ea7d 2024-11-11T03:42:23,020 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/C of d3a8571217c3f763e5d9d9614d29758b into ffeb90c21ac44132888c34aaf029ea7d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:23,020 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:23,020 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/C, priority=13, startTime=1731296542178; duration=0sec 2024-11-11T03:42:23,020 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:23,020 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:C 2024-11-11T03:42:23,091 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/35956794e04f49d297be9e1b8321676a 2024-11-11T03:42:23,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/0937fa8a2e344a6da5fd48ac04300353 is 50, key is test_row_0/B:col10/1731296541374/Put/seqid=0 2024-11-11T03:42:23,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742364_1540 (size=12151) 2024-11-11T03:42:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296603171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:23,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296603171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:23,499 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/0937fa8a2e344a6da5fd48ac04300353 2024-11-11T03:42:23,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/f0434354192c42cc99cc5f35f16ab935 is 50, key is test_row_0/C:col10/1731296541374/Put/seqid=0 2024-11-11T03:42:23,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742365_1541 (size=12151) 2024-11-11T03:42:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T03:42:23,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296603681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:23,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296603681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:23,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T03:42:23,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T03:42:23,908 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/f0434354192c42cc99cc5f35f16ab935 2024-11-11T03:42:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/35956794e04f49d297be9e1b8321676a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/35956794e04f49d297be9e1b8321676a 2024-11-11T03:42:23,914 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/35956794e04f49d297be9e1b8321676a, entries=150, sequenceid=135, filesize=30.4 K 2024-11-11T03:42:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/0937fa8a2e344a6da5fd48ac04300353 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0937fa8a2e344a6da5fd48ac04300353 2024-11-11T03:42:23,917 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0937fa8a2e344a6da5fd48ac04300353, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T03:42:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/f0434354192c42cc99cc5f35f16ab935 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f0434354192c42cc99cc5f35f16ab935 2024-11-11T03:42:23,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f0434354192c42cc99cc5f35f16ab935, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T03:42:23,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d3a8571217c3f763e5d9d9614d29758b in 1645ms, sequenceid=135, compaction requested=false 2024-11-11T03:42:23,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:23,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:23,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-11T03:42:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-11T03:42:23,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T03:42:23,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4080 sec 2024-11-11T03:42:23,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.4120 sec 2024-11-11T03:42:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:24,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:42:24,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:24,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:24,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:24,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:24,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:24,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:24,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111177d6a0a0de5a43859348992a3878d3f7_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296542541/Put/seqid=0 2024-11-11T03:42:24,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742366_1542 (size=12304) 2024-11-11T03:42:24,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296604725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:24,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296604726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296604831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296604831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:25,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296605041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:25,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296605041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:25,102 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:25,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111177d6a0a0de5a43859348992a3878d3f7_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111177d6a0a0de5a43859348992a3878d3f7_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:25,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/8deef19dab414aafb2b4a146f27341d7, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:25,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/8deef19dab414aafb2b4a146f27341d7 is 175, key is test_row_0/A:col10/1731296542541/Put/seqid=0 2024-11-11T03:42:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742367_1543 (size=31105) 2024-11-11T03:42:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296605352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296605352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:25,510 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/8deef19dab414aafb2b4a146f27341d7 2024-11-11T03:42:25,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/632328cf448f417582e4b64733a046e9 is 50, key is test_row_0/B:col10/1731296542541/Put/seqid=0 2024-11-11T03:42:25,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742368_1544 (size=12151) 2024-11-11T03:42:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T03:42:25,682 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:25,683 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-11T03:42:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T03:42:25,684 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:25,685 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:25,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T03:42:25,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T03:42:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:25,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296605862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296605862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:25,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/632328cf448f417582e4b64733a046e9 2024-11-11T03:42:25,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/8739f03e2450489fb10f257d1a9e2801 is 50, key is test_row_0/C:col10/1731296542541/Put/seqid=0 2024-11-11T03:42:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742369_1545 (size=12151) 2024-11-11T03:42:25,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T03:42:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:25,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:25,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T03:42:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296606102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:26,105 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:26,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296606132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:26,135 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:26,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T03:42:26,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:26,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:26,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:26,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296606171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:26,175 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:26,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T03:42:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T03:42:26,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/8739f03e2450489fb10f257d1a9e2801 2024-11-11T03:42:26,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/8deef19dab414aafb2b4a146f27341d7 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/8deef19dab414aafb2b4a146f27341d7 2024-11-11T03:42:26,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/8deef19dab414aafb2b4a146f27341d7, entries=150, sequenceid=153, filesize=30.4 K 2024-11-11T03:42:26,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/632328cf448f417582e4b64733a046e9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/632328cf448f417582e4b64733a046e9 2024-11-11T03:42:26,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/632328cf448f417582e4b64733a046e9, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T03:42:26,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/8739f03e2450489fb10f257d1a9e2801 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/8739f03e2450489fb10f257d1a9e2801 2024-11-11T03:42:26,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/8739f03e2450489fb10f257d1a9e2801, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T03:42:26,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d3a8571217c3f763e5d9d9614d29758b in 1649ms, sequenceid=153, compaction requested=true 2024-11-11T03:42:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:26,342 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:26,342 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:26,343 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:26,343 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:26,343 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/B is initiating minor compaction (all files) 2024-11-11T03:42:26,343 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/A is initiating minor compaction (all files) 2024-11-11T03:42:26,343 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/A in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,343 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/B in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,343 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6a81c306f0dd4db995adb8f30136bec1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0937fa8a2e344a6da5fd48ac04300353, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/632328cf448f417582e4b64733a046e9] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=35.7 K 2024-11-11T03:42:26,343 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/e8c55d006f7d4a2980b9beb9c3a9ed59, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/35956794e04f49d297be9e1b8321676a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/8deef19dab414aafb2b4a146f27341d7] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=91.2 K 2024-11-11T03:42:26,343 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,344 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/e8c55d006f7d4a2980b9beb9c3a9ed59, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/35956794e04f49d297be9e1b8321676a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/8deef19dab414aafb2b4a146f27341d7] 2024-11-11T03:42:26,344 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a81c306f0dd4db995adb8f30136bec1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296540699 2024-11-11T03:42:26,344 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c55d006f7d4a2980b9beb9c3a9ed59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296540699 2024-11-11T03:42:26,344 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0937fa8a2e344a6da5fd48ac04300353, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296541363 2024-11-11T03:42:26,344 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35956794e04f49d297be9e1b8321676a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296541363 2024-11-11T03:42:26,344 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 632328cf448f417582e4b64733a046e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731296542536 2024-11-11T03:42:26,344 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8deef19dab414aafb2b4a146f27341d7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731296542536 2024-11-11T03:42:26,349 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:26,351 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#B#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:26,351 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111693c4aa2e26c4fbb928c8e6017791371_d3a8571217c3f763e5d9d9614d29758b store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:26,351 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/11d8c1f06dd0428f92840e3719bb82c0 is 50, key is test_row_0/B:col10/1731296542541/Put/seqid=0 2024-11-11T03:42:26,353 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111693c4aa2e26c4fbb928c8e6017791371_d3a8571217c3f763e5d9d9614d29758b, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:26,353 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111693c4aa2e26c4fbb928c8e6017791371_d3a8571217c3f763e5d9d9614d29758b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:26,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742371_1547 (size=4469) 2024-11-11T03:42:26,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742370_1546 (size=12459) 2024-11-11T03:42:26,371 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/11d8c1f06dd0428f92840e3719bb82c0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/11d8c1f06dd0428f92840e3719bb82c0 2024-11-11T03:42:26,376 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/B of d3a8571217c3f763e5d9d9614d29758b into 11d8c1f06dd0428f92840e3719bb82c0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:26,376 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:26,376 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/B, priority=13, startTime=1731296546342; duration=0sec 2024-11-11T03:42:26,376 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:26,376 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:B 2024-11-11T03:42:26,376 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:26,377 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:26,377 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/C is initiating minor compaction (all files) 2024-11-11T03:42:26,377 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/C in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,377 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ffeb90c21ac44132888c34aaf029ea7d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f0434354192c42cc99cc5f35f16ab935, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/8739f03e2450489fb10f257d1a9e2801] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=35.7 K 2024-11-11T03:42:26,377 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting ffeb90c21ac44132888c34aaf029ea7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731296540699 2024-11-11T03:42:26,377 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f0434354192c42cc99cc5f35f16ab935, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731296541363 2024-11-11T03:42:26,378 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 8739f03e2450489fb10f257d1a9e2801, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731296542536 2024-11-11T03:42:26,389 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:26,389 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/f8cfb4671863495d9ff138461e02b012 is 50, key is test_row_0/C:col10/1731296542541/Put/seqid=0 2024-11-11T03:42:26,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742372_1548 (size=12459) 2024-11-11T03:42:26,397 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/f8cfb4671863495d9ff138461e02b012 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f8cfb4671863495d9ff138461e02b012 2024-11-11T03:42:26,400 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/C of d3a8571217c3f763e5d9d9614d29758b into f8cfb4671863495d9ff138461e02b012(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:26,400 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:26,400 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/C, priority=13, startTime=1731296546342; duration=0sec 2024-11-11T03:42:26,401 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:26,401 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:C 2024-11-11T03:42:26,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T03:42:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:26,447 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:42:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111740587164e0d4bb5a6b57de27e2923f4_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296544713/Put/seqid=0 2024-11-11T03:42:26,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742373_1549 (size=12304) 2024-11-11T03:42:26,765 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:26,766 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/4d8f000a49c945c18a10363bc7e83708 is 175, key is test_row_0/A:col10/1731296542541/Put/seqid=0 2024-11-11T03:42:26,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742374_1550 (size=31413) 2024-11-11T03:42:26,771 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/4d8f000a49c945c18a10363bc7e83708 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/4d8f000a49c945c18a10363bc7e83708 2024-11-11T03:42:26,774 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/A of d3a8571217c3f763e5d9d9614d29758b into 4d8f000a49c945c18a10363bc7e83708(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:26,774 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:26,774 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/A, priority=13, startTime=1731296546342; duration=0sec 2024-11-11T03:42:26,774 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:26,774 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:A 2024-11-11T03:42:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T03:42:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:26,858 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111740587164e0d4bb5a6b57de27e2923f4_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111740587164e0d4bb5a6b57de27e2923f4_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:26,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/c59a6060756547d194d3878481d58cf8, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:26,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/c59a6060756547d194d3878481d58cf8 is 175, key is test_row_0/A:col10/1731296544713/Put/seqid=0 2024-11-11T03:42:26,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742375_1551 (size=31105) 2024-11-11T03:42:26,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296606905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296606906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:27,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296607012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:27,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296607012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:27,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296607221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:27,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:27,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296607222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:27,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:27,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:27,263 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/c59a6060756547d194d3878481d58cf8 2024-11-11T03:42:27,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/3fec0de11e744e2988f2b89f8cce002c is 50, key is test_row_0/B:col10/1731296544713/Put/seqid=0 2024-11-11T03:42:27,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742376_1552 (size=12151) 2024-11-11T03:42:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296607532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:27,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296607532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:27,671 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/3fec0de11e744e2988f2b89f8cce002c 2024-11-11T03:42:27,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b4088b5a24cd439d9114e55ed38931a1 is 50, key is test_row_0/C:col10/1731296544713/Put/seqid=0 2024-11-11T03:42:27,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742377_1553 (size=12151) 2024-11-11T03:42:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T03:42:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296608041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296608042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:28,080 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b4088b5a24cd439d9114e55ed38931a1 2024-11-11T03:42:28,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/c59a6060756547d194d3878481d58cf8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/c59a6060756547d194d3878481d58cf8 2024-11-11T03:42:28,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/c59a6060756547d194d3878481d58cf8, entries=150, sequenceid=172, filesize=30.4 K 2024-11-11T03:42:28,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/3fec0de11e744e2988f2b89f8cce002c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3fec0de11e744e2988f2b89f8cce002c 2024-11-11T03:42:28,089 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3fec0de11e744e2988f2b89f8cce002c, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T03:42:28,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b4088b5a24cd439d9114e55ed38931a1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b4088b5a24cd439d9114e55ed38931a1 2024-11-11T03:42:28,092 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b4088b5a24cd439d9114e55ed38931a1, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T03:42:28,093 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d3a8571217c3f763e5d9d9614d29758b in 1645ms, sequenceid=172, compaction requested=false 2024-11-11T03:42:28,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:28,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:28,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-11T03:42:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-11T03:42:28,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T03:42:28,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4090 sec 2024-11-11T03:42:28,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.4130 sec 2024-11-11T03:42:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:29,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T03:42:29,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:29,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:29,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:29,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:29,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:29,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111164ea573d9851475a9c74c184d9e6d571_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296546904/Put/seqid=0 2024-11-11T03:42:29,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742378_1554 (size=12304) 2024-11-11T03:42:29,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:29,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111164ea573d9851475a9c74c184d9e6d571_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164ea573d9851475a9c74c184d9e6d571_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:29,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/9423cbb989fb460ba9983436c202f555, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:29,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/9423cbb989fb460ba9983436c202f555 is 175, key is test_row_0/A:col10/1731296546904/Put/seqid=0 2024-11-11T03:42:29,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742379_1555 (size=31105) 2024-11-11T03:42:29,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296609088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296609091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296609202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296609202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296609412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296609412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,478 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/9423cbb989fb460ba9983436c202f555 2024-11-11T03:42:29,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/078adc1b22c84a78a1af7cdc55ce68fd is 50, key is test_row_0/B:col10/1731296546904/Put/seqid=0 2024-11-11T03:42:29,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742380_1556 (size=12151) 2024-11-11T03:42:29,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/078adc1b22c84a78a1af7cdc55ce68fd 2024-11-11T03:42:29,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0a7fd657eac94e9aacdc40db472872de is 50, key is test_row_0/C:col10/1731296546904/Put/seqid=0 2024-11-11T03:42:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742381_1557 (size=12151) 2024-11-11T03:42:29,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296609722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296609722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T03:42:29,852 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:29,853 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:29,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0a7fd657eac94e9aacdc40db472872de 2024-11-11T03:42:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-11T03:42:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T03:42:29,900 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:29,900 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:29,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:29,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/9423cbb989fb460ba9983436c202f555 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/9423cbb989fb460ba9983436c202f555 2024-11-11T03:42:29,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/9423cbb989fb460ba9983436c202f555, entries=150, sequenceid=189, filesize=30.4 K 2024-11-11T03:42:29,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/078adc1b22c84a78a1af7cdc55ce68fd as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/078adc1b22c84a78a1af7cdc55ce68fd 2024-11-11T03:42:29,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/078adc1b22c84a78a1af7cdc55ce68fd, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T03:42:29,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0a7fd657eac94e9aacdc40db472872de as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a7fd657eac94e9aacdc40db472872de 2024-11-11T03:42:29,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a7fd657eac94e9aacdc40db472872de, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T03:42:29,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d3a8571217c3f763e5d9d9614d29758b in 858ms, sequenceid=189, compaction requested=true 2024-11-11T03:42:29,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:29,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:29,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:29,911 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:29,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:29,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:29,911 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:29,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:29,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/B is initiating minor compaction (all files) 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/A is initiating minor compaction (all files) 2024-11-11T03:42:29,912 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/A in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:29,912 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/B in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:29,912 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/4d8f000a49c945c18a10363bc7e83708, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/c59a6060756547d194d3878481d58cf8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/9423cbb989fb460ba9983436c202f555] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=91.4 K 2024-11-11T03:42:29,912 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/11d8c1f06dd0428f92840e3719bb82c0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3fec0de11e744e2988f2b89f8cce002c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/078adc1b22c84a78a1af7cdc55ce68fd] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=35.9 K 2024-11-11T03:42:29,912 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/4d8f000a49c945c18a10363bc7e83708, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/c59a6060756547d194d3878481d58cf8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/9423cbb989fb460ba9983436c202f555] 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8f000a49c945c18a10363bc7e83708, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731296542536 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d8c1f06dd0428f92840e3719bb82c0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731296542536 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting c59a6060756547d194d3878481d58cf8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731296544708 2024-11-11T03:42:29,912 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fec0de11e744e2988f2b89f8cce002c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731296544708 2024-11-11T03:42:29,913 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9423cbb989fb460ba9983436c202f555, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296546904 2024-11-11T03:42:29,913 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 078adc1b22c84a78a1af7cdc55ce68fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296546904 2024-11-11T03:42:29,918 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#B#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:29,918 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/0e7d8f4335844e688ba004763aa638c5 is 50, key is test_row_0/B:col10/1731296546904/Put/seqid=0 2024-11-11T03:42:29,920 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:29,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742382_1558 (size=12561) 2024-11-11T03:42:29,923 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111168c9d6cab05742b7bee967d64a9848c7_d3a8571217c3f763e5d9d9614d29758b store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:29,925 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111168c9d6cab05742b7bee967d64a9848c7_d3a8571217c3f763e5d9d9614d29758b, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:29,925 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111168c9d6cab05742b7bee967d64a9848c7_d3a8571217c3f763e5d9d9614d29758b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:29,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742383_1559 (size=4469) 2024-11-11T03:42:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T03:42:30,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T03:42:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:30,053 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T03:42:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:30,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:30,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111195b2554c2c2c4f3fb2f9f92d697eeb2e_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296549087/Put/seqid=0 2024-11-11T03:42:30,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742384_1560 (size=12304) 2024-11-11T03:42:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T03:42:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:30,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296610261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296610264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:30,326 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/0e7d8f4335844e688ba004763aa638c5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0e7d8f4335844e688ba004763aa638c5 2024-11-11T03:42:30,329 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:30,329 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/B of d3a8571217c3f763e5d9d9614d29758b into 0e7d8f4335844e688ba004763aa638c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:30,329 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:30,329 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/B, priority=13, startTime=1731296549911; duration=0sec 2024-11-11T03:42:30,329 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:30,329 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:B 2024-11-11T03:42:30,329 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:30,329 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/fffc9b47127b47308ba6b0f39ecd57ce is 175, key is test_row_0/A:col10/1731296546904/Put/seqid=0 2024-11-11T03:42:30,330 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:30,330 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/C is initiating minor compaction (all files) 2024-11-11T03:42:30,330 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/C in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:30,330 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f8cfb4671863495d9ff138461e02b012, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b4088b5a24cd439d9114e55ed38931a1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a7fd657eac94e9aacdc40db472872de] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=35.9 K 2024-11-11T03:42:30,330 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting f8cfb4671863495d9ff138461e02b012, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731296542536 2024-11-11T03:42:30,331 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b4088b5a24cd439d9114e55ed38931a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731296544708 2024-11-11T03:42:30,331 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7fd657eac94e9aacdc40db472872de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296546904 2024-11-11T03:42:30,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742385_1561 (size=31515) 2024-11-11T03:42:30,336 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#C#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:30,337 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/db24ff949a01484db1561292d0d8babf is 50, key is test_row_0/C:col10/1731296546904/Put/seqid=0 2024-11-11T03:42:30,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742386_1562 (size=12561) 2024-11-11T03:42:30,342 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/db24ff949a01484db1561292d0d8babf as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/db24ff949a01484db1561292d0d8babf 2024-11-11T03:42:30,345 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/C of d3a8571217c3f763e5d9d9614d29758b into db24ff949a01484db1561292d0d8babf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:30,345 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:30,345 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/C, priority=13, startTime=1731296549911; duration=0sec 2024-11-11T03:42:30,345 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:30,345 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:C 2024-11-11T03:42:30,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296610372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296610372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:30,465 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111195b2554c2c2c4f3fb2f9f92d697eeb2e_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195b2554c2c2c4f3fb2f9f92d697eeb2e_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:30,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/f1b54a4ae84349d1a20b0539c8a349ff, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:30,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/f1b54a4ae84349d1a20b0539c8a349ff is 175, key is test_row_0/A:col10/1731296549087/Put/seqid=0 2024-11-11T03:42:30,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742387_1563 (size=31105) 2024-11-11T03:42:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T03:42:30,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296610582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296610582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:30,736 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/fffc9b47127b47308ba6b0f39ecd57ce as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fffc9b47127b47308ba6b0f39ecd57ce 2024-11-11T03:42:30,739 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/A of d3a8571217c3f763e5d9d9614d29758b into fffc9b47127b47308ba6b0f39ecd57ce(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:30,739 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:30,739 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/A, priority=13, startTime=1731296549911; duration=0sec 2024-11-11T03:42:30,739 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:30,739 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:A 2024-11-11T03:42:30,873 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/f1b54a4ae84349d1a20b0539c8a349ff 2024-11-11T03:42:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/5201a47c179d4aa5be73bff345cbef0d is 50, key is test_row_0/B:col10/1731296549087/Put/seqid=0 2024-11-11T03:42:30,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742388_1564 (size=12151) 2024-11-11T03:42:30,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296610891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296610891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T03:42:31,282 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/5201a47c179d4aa5be73bff345cbef0d 2024-11-11T03:42:31,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/a3f1c390e881491abdcc66ba19c8596e is 50, key is test_row_0/C:col10/1731296549087/Put/seqid=0 2024-11-11T03:42:31,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742389_1565 (size=12151) 2024-11-11T03:42:31,290 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/a3f1c390e881491abdcc66ba19c8596e 2024-11-11T03:42:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/f1b54a4ae84349d1a20b0539c8a349ff as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f1b54a4ae84349d1a20b0539c8a349ff 2024-11-11T03:42:31,296 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f1b54a4ae84349d1a20b0539c8a349ff, entries=150, sequenceid=205, filesize=30.4 K 2024-11-11T03:42:31,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/5201a47c179d4aa5be73bff345cbef0d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/5201a47c179d4aa5be73bff345cbef0d 2024-11-11T03:42:31,299 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/5201a47c179d4aa5be73bff345cbef0d, entries=150, sequenceid=205, filesize=11.9 K 2024-11-11T03:42:31,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/a3f1c390e881491abdcc66ba19c8596e as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/a3f1c390e881491abdcc66ba19c8596e 2024-11-11T03:42:31,302 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/a3f1c390e881491abdcc66ba19c8596e, entries=150, sequenceid=205, filesize=11.9 K 2024-11-11T03:42:31,303 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d3a8571217c3f763e5d9d9614d29758b in 1250ms, sequenceid=205, compaction requested=false 2024-11-11T03:42:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-11T03:42:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-11T03:42:31,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T03:42:31,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-11T03:42:31,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4530 sec 2024-11-11T03:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:31,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T03:42:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:31,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0bc275ab95247308b269a9dacfcdf0d_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296551403/Put/seqid=0 2024-11-11T03:42:31,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742390_1566 (size=12304) 2024-11-11T03:42:31,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296611427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:31,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296611433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296611541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296611542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:31,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296611752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:31,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296611752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:31,814 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:31,817 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0bc275ab95247308b269a9dacfcdf0d_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0bc275ab95247308b269a9dacfcdf0d_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:31,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/66a2b4bf1823476987e411bf42497e59, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:31,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/66a2b4bf1823476987e411bf42497e59 is 175, key is test_row_0/A:col10/1731296551403/Put/seqid=0 2024-11-11T03:42:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742391_1567 (size=31105) 2024-11-11T03:42:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T03:42:32,041 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:32,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-11T03:42:32,043 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T03:42:32,044 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:32,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:32,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296612061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:32,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296612062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T03:42:32,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T03:42:32,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,221 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/66a2b4bf1823476987e411bf42497e59 2024-11-11T03:42:32,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/edd39481b40e4275aa26a5afa73d21ee is 50, key is test_row_0/B:col10/1731296551403/Put/seqid=0 2024-11-11T03:42:32,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742392_1568 (size=12151) 2024-11-11T03:42:32,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T03:42:32,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:32,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T03:42:32,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T03:42:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296612571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296612572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:32,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/edd39481b40e4275aa26a5afa73d21ee 2024-11-11T03:42:32,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/27fb5928f64c438cba7be5328972c31a is 50, key is test_row_0/C:col10/1731296551403/Put/seqid=0 2024-11-11T03:42:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742393_1569 (size=12151) 2024-11-11T03:42:32,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T03:42:32,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:32,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T03:42:32,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T03:42:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T03:42:32,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:32,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:32,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:32,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:33,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/27fb5928f64c438cba7be5328972c31a 2024-11-11T03:42:33,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/66a2b4bf1823476987e411bf42497e59 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/66a2b4bf1823476987e411bf42497e59 2024-11-11T03:42:33,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/66a2b4bf1823476987e411bf42497e59, entries=150, sequenceid=225, filesize=30.4 K 2024-11-11T03:42:33,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/edd39481b40e4275aa26a5afa73d21ee as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/edd39481b40e4275aa26a5afa73d21ee 2024-11-11T03:42:33,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/edd39481b40e4275aa26a5afa73d21ee, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T03:42:33,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/27fb5928f64c438cba7be5328972c31a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/27fb5928f64c438cba7be5328972c31a 2024-11-11T03:42:33,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/27fb5928f64c438cba7be5328972c31a, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T03:42:33,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d3a8571217c3f763e5d9d9614d29758b in 1647ms, sequenceid=225, compaction requested=true 2024-11-11T03:42:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:33,051 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:33,051 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/B is initiating minor compaction (all files) 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/A is initiating minor compaction (all files) 2024-11-11T03:42:33,052 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/B in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:33,052 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/A in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:33,052 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0e7d8f4335844e688ba004763aa638c5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/5201a47c179d4aa5be73bff345cbef0d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/edd39481b40e4275aa26a5afa73d21ee] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=36.0 K 2024-11-11T03:42:33,052 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fffc9b47127b47308ba6b0f39ecd57ce, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f1b54a4ae84349d1a20b0539c8a349ff, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/66a2b4bf1823476987e411bf42497e59] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=91.5 K 2024-11-11T03:42:33,052 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fffc9b47127b47308ba6b0f39ecd57ce, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f1b54a4ae84349d1a20b0539c8a349ff, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/66a2b4bf1823476987e411bf42497e59] 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7d8f4335844e688ba004763aa638c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296546904 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting fffc9b47127b47308ba6b0f39ecd57ce, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296546904 2024-11-11T03:42:33,052 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 5201a47c179d4aa5be73bff345cbef0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731296549084 2024-11-11T03:42:33,053 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b54a4ae84349d1a20b0539c8a349ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731296549084 2024-11-11T03:42:33,053 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting edd39481b40e4275aa26a5afa73d21ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731296550260 2024-11-11T03:42:33,053 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a2b4bf1823476987e411bf42497e59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731296550260 2024-11-11T03:42:33,057 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:33,070 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d1e942e9b72645e0829795962c711ce4_d3a8571217c3f763e5d9d9614d29758b store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:33,071 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:33,072 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/4a8b0865896646cab48d2e7374c4520d is 50, key is test_row_0/B:col10/1731296551403/Put/seqid=0 2024-11-11T03:42:33,072 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d1e942e9b72645e0829795962c711ce4_d3a8571217c3f763e5d9d9614d29758b, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:33,072 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d1e942e9b72645e0829795962c711ce4_d3a8571217c3f763e5d9d9614d29758b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:33,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742395_1571 (size=4469) 2024-11-11T03:42:33,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742394_1570 (size=12663) 2024-11-11T03:42:33,077 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#A#compaction#479 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:33,077 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/f69af747396f4a1fb8bd398872b39e99 is 175, key is test_row_0/A:col10/1731296551403/Put/seqid=0 2024-11-11T03:42:33,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742396_1572 (size=31617) 2024-11-11T03:42:33,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T03:42:33,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:33,110 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T03:42:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:33,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111184ac84f725e34349b7fb6d8acfd8a1b6_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296551421/Put/seqid=0 2024-11-11T03:42:33,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742397_1573 (size=12304) 2024-11-11T03:42:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T03:42:33,479 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/4a8b0865896646cab48d2e7374c4520d as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/4a8b0865896646cab48d2e7374c4520d 2024-11-11T03:42:33,482 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/B of d3a8571217c3f763e5d9d9614d29758b into 4a8b0865896646cab48d2e7374c4520d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:33,482 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:33,482 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/B, priority=13, startTime=1731296553051; duration=0sec 2024-11-11T03:42:33,483 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:33,483 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:B 2024-11-11T03:42:33,483 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:33,483 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/f69af747396f4a1fb8bd398872b39e99 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f69af747396f4a1fb8bd398872b39e99 2024-11-11T03:42:33,483 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:33,483 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/C is initiating minor compaction (all files) 2024-11-11T03:42:33,483 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/C in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:33,484 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/db24ff949a01484db1561292d0d8babf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/a3f1c390e881491abdcc66ba19c8596e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/27fb5928f64c438cba7be5328972c31a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=36.0 K 2024-11-11T03:42:33,484 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting db24ff949a01484db1561292d0d8babf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731296546904 2024-11-11T03:42:33,484 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f1c390e881491abdcc66ba19c8596e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731296549084 2024-11-11T03:42:33,485 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fb5928f64c438cba7be5328972c31a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731296550260 2024-11-11T03:42:33,487 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/A of d3a8571217c3f763e5d9d9614d29758b into f69af747396f4a1fb8bd398872b39e99(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:33,487 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:33,487 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/A, priority=13, startTime=1731296553051; duration=0sec 2024-11-11T03:42:33,487 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:33,487 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:A 2024-11-11T03:42:33,491 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:33,492 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b0a2c1bdfafc463ebd70853a88551fa3 is 50, key is test_row_0/C:col10/1731296551403/Put/seqid=0 2024-11-11T03:42:33,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742398_1574 (size=12663) 2024-11-11T03:42:33,503 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/b0a2c1bdfafc463ebd70853a88551fa3 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b0a2c1bdfafc463ebd70853a88551fa3 2024-11-11T03:42:33,506 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/C of d3a8571217c3f763e5d9d9614d29758b into b0a2c1bdfafc463ebd70853a88551fa3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:33,506 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:33,506 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/C, priority=13, startTime=1731296553051; duration=0sec 2024-11-11T03:42:33,506 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:33,506 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:C 2024-11-11T03:42:33,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:33,522 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111184ac84f725e34349b7fb6d8acfd8a1b6_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111184ac84f725e34349b7fb6d8acfd8a1b6_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:33,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/43dfb4c090004ec5923555326c576ead, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:33,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/43dfb4c090004ec5923555326c576ead is 175, key is test_row_0/A:col10/1731296551421/Put/seqid=0 2024-11-11T03:42:33,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742399_1575 (size=31105) 2024-11-11T03:42:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:33,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296613608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:33,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296613610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:33,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296613721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:33,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296613721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:33,926 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/43dfb4c090004ec5923555326c576ead 2024-11-11T03:42:33,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/7eb1af0986ef4b288f810494459ae585 is 50, key is test_row_0/B:col10/1731296551421/Put/seqid=0 2024-11-11T03:42:33,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296613932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296613932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:33,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742400_1576 (size=12151) 2024-11-11T03:42:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T03:42:34,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:34,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296614241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:34,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:34,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296614242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:34,336 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/7eb1af0986ef4b288f810494459ae585 2024-11-11T03:42:34,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/c9186ba3d6724700a4a92970e61d5489 is 50, key is test_row_0/C:col10/1731296551421/Put/seqid=0 2024-11-11T03:42:34,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742401_1577 (size=12151) 2024-11-11T03:42:34,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/c9186ba3d6724700a4a92970e61d5489 2024-11-11T03:42:34,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/43dfb4c090004ec5923555326c576ead as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/43dfb4c090004ec5923555326c576ead 2024-11-11T03:42:34,751 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/43dfb4c090004ec5923555326c576ead, entries=150, sequenceid=240, filesize=30.4 K 2024-11-11T03:42:34,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/7eb1af0986ef4b288f810494459ae585 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/7eb1af0986ef4b288f810494459ae585 2024-11-11T03:42:34,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296614751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296614751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:34,754 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/7eb1af0986ef4b288f810494459ae585, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T03:42:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:34,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/c9186ba3d6724700a4a92970e61d5489 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/c9186ba3d6724700a4a92970e61d5489 2024-11-11T03:42:34,757 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/c9186ba3d6724700a4a92970e61d5489, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T03:42:34,758 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d3a8571217c3f763e5d9d9614d29758b in 1648ms, sequenceid=240, compaction requested=false 2024-11-11T03:42:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-11T03:42:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-11T03:42:34,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-11T03:42:34,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7150 sec 2024-11-11T03:42:34,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.7190 sec 2024-11-11T03:42:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:35,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T03:42:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:35,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fe9e5fe3604745daa7c862925cbd7270_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296553604/Put/seqid=0 2024-11-11T03:42:35,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742402_1578 (size=17534) 2024-11-11T03:42:35,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296615830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296615831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:35,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296615942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:35,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296615942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:35,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46840 deadline: 1731296616152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296616152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296616152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,155 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_0' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:36,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T03:42:36,202 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:36,202 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T03:42:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-11T03:42:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T03:42:36,203 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T03:42:36,204 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T03:42:36,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T03:42:36,217 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:36,220 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fe9e5fe3604745daa7c862925cbd7270_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fe9e5fe3604745daa7c862925cbd7270_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:36,220 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/09b2e6c1464c498c9dac2cc9455783b9, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:36,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/09b2e6c1464c498c9dac2cc9455783b9 is 175, key is test_row_0/A:col10/1731296553604/Put/seqid=0 2024-11-11T03:42:36,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742403_1579 (size=48639) 2024-11-11T03:42:36,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46884 deadline: 1731296616242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:36,245 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18315 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46854 deadline: 1731296616262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:36,265 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3ffe7f9d089e,34473,1731296384288 for 'test_row_2' in d3a8571217c3f763e5d9d9614d29758b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18328 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T03:42:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T03:42:36,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T03:42:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296616462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296616462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:36,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T03:42:36,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:36,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:42:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T03:42:36,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/09b2e6c1464c498c9dac2cc9455783b9 2024-11-11T03:42:36,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/58b8d26478824692b09ca57ffb028036 is 50, key is test_row_0/B:col10/1731296553604/Put/seqid=0 2024-11-11T03:42:36,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742404_1580 (size=12301) 2024-11-11T03:42:36,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/58b8d26478824692b09ca57ffb028036 2024-11-11T03:42:36,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/e6320f2736ea4392a32996a169ebc288 is 50, key is test_row_0/C:col10/1731296553604/Put/seqid=0 2024-11-11T03:42:36,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742405_1581 (size=12301) 2024-11-11T03:42:36,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/e6320f2736ea4392a32996a169ebc288 2024-11-11T03:42:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/09b2e6c1464c498c9dac2cc9455783b9 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/09b2e6c1464c498c9dac2cc9455783b9 2024-11-11T03:42:36,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/09b2e6c1464c498c9dac2cc9455783b9, entries=250, sequenceid=262, filesize=47.5 K 2024-11-11T03:42:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/58b8d26478824692b09ca57ffb028036 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/58b8d26478824692b09ca57ffb028036 2024-11-11T03:42:36,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/58b8d26478824692b09ca57ffb028036, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T03:42:36,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/e6320f2736ea4392a32996a169ebc288 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e6320f2736ea4392a32996a169ebc288 2024-11-11T03:42:36,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e6320f2736ea4392a32996a169ebc288, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T03:42:36,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d3a8571217c3f763e5d9d9614d29758b in 847ms, sequenceid=262, compaction requested=true 2024-11-11T03:42:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:36,654 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:36,654 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:36,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111361 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/B is initiating minor compaction (all files) 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/A is initiating minor compaction (all files) 2024-11-11T03:42:36,655 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/A in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,655 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/B in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,655 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/4a8b0865896646cab48d2e7374c4520d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/7eb1af0986ef4b288f810494459ae585, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/58b8d26478824692b09ca57ffb028036] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=36.2 K 2024-11-11T03:42:36,655 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f69af747396f4a1fb8bd398872b39e99, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/43dfb4c090004ec5923555326c576ead, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/09b2e6c1464c498c9dac2cc9455783b9] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=108.8 K 2024-11-11T03:42:36,655 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f69af747396f4a1fb8bd398872b39e99, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/43dfb4c090004ec5923555326c576ead, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/09b2e6c1464c498c9dac2cc9455783b9] 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a8b0865896646cab48d2e7374c4520d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731296550260 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69af747396f4a1fb8bd398872b39e99, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731296550260 2024-11-11T03:42:36,655 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eb1af0986ef4b288f810494459ae585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731296551421 2024-11-11T03:42:36,656 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43dfb4c090004ec5923555326c576ead, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731296551421 2024-11-11T03:42:36,656 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 58b8d26478824692b09ca57ffb028036, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296553604 2024-11-11T03:42:36,656 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09b2e6c1464c498c9dac2cc9455783b9, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296553604 2024-11-11T03:42:36,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T03:42:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:36,661 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T03:42:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:36,672 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:36,674 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:36,674 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/e31d5e793a8542c598bb14678e05531f is 50, key is test_row_0/B:col10/1731296553604/Put/seqid=0 2024-11-11T03:42:36,674 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111701ea4f0a26741d4806e43ec70834f47_d3a8571217c3f763e5d9d9614d29758b store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b1158207b18c4a8eaa912c64edc90b67_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296555830/Put/seqid=0 2024-11-11T03:42:36,677 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111701ea4f0a26741d4806e43ec70834f47_d3a8571217c3f763e5d9d9614d29758b, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:36,677 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111701ea4f0a26741d4806e43ec70834f47_d3a8571217c3f763e5d9d9614d29758b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:36,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742406_1582 (size=12915) 2024-11-11T03:42:36,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742407_1583 (size=12454) 2024-11-11T03:42:36,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742408_1584 (size=4469) 2024-11-11T03:42:36,701 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#A#compaction#489 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:36,701 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/b37eaee2b3a74837b8d654936625b500 is 175, key is test_row_0/A:col10/1731296553604/Put/seqid=0 2024-11-11T03:42:36,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742409_1585 (size=31869) 2024-11-11T03:42:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T03:42:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:36,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. as already flushing 2024-11-11T03:42:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296617012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296617014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,091 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/e31d5e793a8542c598bb14678e05531f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e31d5e793a8542c598bb14678e05531f 2024-11-11T03:42:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:37,094 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/B of d3a8571217c3f763e5d9d9614d29758b into e31d5e793a8542c598bb14678e05531f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:37,094 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:37,094 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/B, priority=13, startTime=1731296556654; duration=0sec 2024-11-11T03:42:37,094 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:37,094 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:B 2024-11-11T03:42:37,094 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:37,095 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:37,095 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/C is initiating minor compaction (all files) 2024-11-11T03:42:37,095 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/C in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:37,095 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b0a2c1bdfafc463ebd70853a88551fa3, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/c9186ba3d6724700a4a92970e61d5489, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e6320f2736ea4392a32996a169ebc288] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=36.2 K 2024-11-11T03:42:37,095 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b1158207b18c4a8eaa912c64edc90b67_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b1158207b18c4a8eaa912c64edc90b67_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:37,096 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a2c1bdfafc463ebd70853a88551fa3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731296550260 2024-11-11T03:42:37,096 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting c9186ba3d6724700a4a92970e61d5489, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731296551421 2024-11-11T03:42:37,096 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e6320f2736ea4392a32996a169ebc288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296553604 2024-11-11T03:42:37,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/81db6c4efd6f4a44ab91fb1437844ca1, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:37,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/81db6c4efd6f4a44ab91fb1437844ca1 is 175, key is test_row_0/A:col10/1731296555830/Put/seqid=0 2024-11-11T03:42:37,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742410_1586 (size=31255) 2024-11-11T03:42:37,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/81db6c4efd6f4a44ab91fb1437844ca1 2024-11-11T03:42:37,101 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:37,102 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/6030fa6c5d1e40baba6069a96382f387 is 50, key is test_row_0/C:col10/1731296553604/Put/seqid=0 2024-11-11T03:42:37,115 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/b37eaee2b3a74837b8d654936625b500 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/b37eaee2b3a74837b8d654936625b500 2024-11-11T03:42:37,119 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/A of d3a8571217c3f763e5d9d9614d29758b into b37eaee2b3a74837b8d654936625b500(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:37,119 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:37,119 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/A, priority=13, startTime=1731296556654; duration=0sec 2024-11-11T03:42:37,119 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:37,119 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:A 2024-11-11T03:42:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/2e3edeb2bdfd4b78909894ccea3082a0 is 50, key is test_row_0/B:col10/1731296555830/Put/seqid=0 2024-11-11T03:42:37,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742411_1587 (size=12915) 2024-11-11T03:42:37,125 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/6030fa6c5d1e40baba6069a96382f387 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/6030fa6c5d1e40baba6069a96382f387 2024-11-11T03:42:37,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296617122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296617122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,129 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/C of d3a8571217c3f763e5d9d9614d29758b into 6030fa6c5d1e40baba6069a96382f387(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:37,129 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:37,129 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/C, priority=13, startTime=1731296556654; duration=0sec 2024-11-11T03:42:37,129 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:37,129 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:C 2024-11-11T03:42:37,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742412_1588 (size=12301) 2024-11-11T03:42:37,217 INFO [Thread-2294 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2294. 2024-11-11T03:42:37,217 DEBUG [Thread-2294 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:37,217 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,217 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:37,218 INFO [Thread-2296 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2296. 2024-11-11T03:42:37,218 DEBUG [Thread-2296 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:37,218 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,218 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:37,218 INFO [Thread-2292 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2292. 2024-11-11T03:42:37,218 DEBUG [Thread-2292 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:37,218 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,218 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:37,221 INFO [Thread-2290 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2290. 2024-11-11T03:42:37,221 DEBUG [Thread-2290 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:37,221 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,221 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:37,222 INFO [Thread-2298 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2298. 2024-11-11T03:42:37,222 DEBUG [Thread-2298 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:37,222 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,222 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:37,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296617332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296617332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T03:42:37,533 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/2e3edeb2bdfd4b78909894ccea3082a0 2024-11-11T03:42:37,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/028705d5904c43959926b50416b4a175 is 50, key is test_row_0/C:col10/1731296555830/Put/seqid=0 2024-11-11T03:42:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742413_1589 (size=12301) 2024-11-11T03:42:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T03:42:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1731296617642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1731296617642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 , the old value is region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T03:42:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., hostname=3ffe7f9d089e,34473,1731296384288, seqNum=5 because the exception is null or not the one we care about 2024-11-11T03:42:37,955 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/028705d5904c43959926b50416b4a175 2024-11-11T03:42:37,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/81db6c4efd6f4a44ab91fb1437844ca1 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/81db6c4efd6f4a44ab91fb1437844ca1 2024-11-11T03:42:37,971 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/81db6c4efd6f4a44ab91fb1437844ca1, entries=150, sequenceid=275, filesize=30.5 K 2024-11-11T03:42:37,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/2e3edeb2bdfd4b78909894ccea3082a0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/2e3edeb2bdfd4b78909894ccea3082a0 2024-11-11T03:42:37,976 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/2e3edeb2bdfd4b78909894ccea3082a0, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T03:42:37,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/028705d5904c43959926b50416b4a175 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/028705d5904c43959926b50416b4a175 2024-11-11T03:42:37,981 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/028705d5904c43959926b50416b4a175, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T03:42:37,982 INFO [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d3a8571217c3f763e5d9d9614d29758b in 1320ms, sequenceid=275, compaction requested=false 2024-11-11T03:42:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ffe7f9d089e:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-11T03:42:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-11T03:42:37,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-11T03:42:37,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7790 sec 2024-11-11T03:42:37,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.7840 sec 2024-11-11T03:42:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34473 {}] regionserver.HRegion(8855): Flush requested on d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:38,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T03:42:38,158 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-11T03:42:38,158 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:38,158 INFO [Thread-2281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2281. 2024-11-11T03:42:38,158 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:38,158 DEBUG [Thread-2281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:38,158 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:38,158 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:38,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:38,158 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:38,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:38,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T03:42:38,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111189f93b77a966405288b91e354a408609_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296558153/Put/seqid=0 2024-11-11T03:42:38,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742414_1590 (size=12454) 2024-11-11T03:42:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T03:42:38,354 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:38,570 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:38,579 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111189f93b77a966405288b91e354a408609_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111189f93b77a966405288b91e354a408609_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:38,581 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/50c1c3dab2484d5aa28562b75f7a7b9a, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:38,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/50c1c3dab2484d5aa28562b75f7a7b9a is 175, key is test_row_0/A:col10/1731296558153/Put/seqid=0 2024-11-11T03:42:38,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742415_1591 (size=31255) 2024-11-11T03:42:38,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/50c1c3dab2484d5aa28562b75f7a7b9a 2024-11-11T03:42:39,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/54f8ac459c34481e8340e39b85eb3452 is 50, key is test_row_0/B:col10/1731296558153/Put/seqid=0 2024-11-11T03:42:39,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742416_1592 (size=12301) 2024-11-11T03:42:39,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/54f8ac459c34481e8340e39b85eb3452 2024-11-11T03:42:39,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/57414df99f5943c9a4c0aac569321046 is 50, key is test_row_0/C:col10/1731296558153/Put/seqid=0 2024-11-11T03:42:39,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742417_1593 (size=12301) 2024-11-11T03:42:39,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/57414df99f5943c9a4c0aac569321046 2024-11-11T03:42:39,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/50c1c3dab2484d5aa28562b75f7a7b9a as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/50c1c3dab2484d5aa28562b75f7a7b9a 2024-11-11T03:42:39,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/50c1c3dab2484d5aa28562b75f7a7b9a, entries=150, sequenceid=299, filesize=30.5 K 2024-11-11T03:42:39,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/54f8ac459c34481e8340e39b85eb3452 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/54f8ac459c34481e8340e39b85eb3452 2024-11-11T03:42:39,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/54f8ac459c34481e8340e39b85eb3452, entries=150, sequenceid=299, filesize=12.0 K 2024-11-11T03:42:39,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/57414df99f5943c9a4c0aac569321046 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/57414df99f5943c9a4c0aac569321046 2024-11-11T03:42:39,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/57414df99f5943c9a4c0aac569321046, entries=150, sequenceid=299, filesize=12.0 K 2024-11-11T03:42:39,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for d3a8571217c3f763e5d9d9614d29758b in 1700ms, sequenceid=299, compaction requested=true 2024-11-11T03:42:39,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:39,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T03:42:39,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:39,857 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:39,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T03:42:39,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:39,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d3a8571217c3f763e5d9d9614d29758b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T03:42:39,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:39,857 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:39,858 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:39,858 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:39,858 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/A is initiating minor compaction (all files) 2024-11-11T03:42:39,858 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/B is initiating minor compaction (all files) 2024-11-11T03:42:39,858 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/B in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:39,858 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/A in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:39,858 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e31d5e793a8542c598bb14678e05531f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/2e3edeb2bdfd4b78909894ccea3082a0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/54f8ac459c34481e8340e39b85eb3452] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=36.6 K 2024-11-11T03:42:39,858 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/b37eaee2b3a74837b8d654936625b500, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/81db6c4efd6f4a44ab91fb1437844ca1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/50c1c3dab2484d5aa28562b75f7a7b9a] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=92.2 K 2024-11-11T03:42:39,858 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:39,859 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. files: [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/b37eaee2b3a74837b8d654936625b500, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/81db6c4efd6f4a44ab91fb1437844ca1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/50c1c3dab2484d5aa28562b75f7a7b9a] 2024-11-11T03:42:39,859 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting e31d5e793a8542c598bb14678e05531f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296553604 2024-11-11T03:42:39,859 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting b37eaee2b3a74837b8d654936625b500, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296553604 2024-11-11T03:42:39,859 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3edeb2bdfd4b78909894ccea3082a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731296555811 2024-11-11T03:42:39,859 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81db6c4efd6f4a44ab91fb1437844ca1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731296555811 2024-11-11T03:42:39,860 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f8ac459c34481e8340e39b85eb3452, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731296557008 2024-11-11T03:42:39,860 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c1c3dab2484d5aa28562b75f7a7b9a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731296557008 2024-11-11T03:42:39,867 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:39,867 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:39,868 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/932cb6b9bd394925aab40422781202c8 is 50, key is test_row_0/B:col10/1731296558153/Put/seqid=0 2024-11-11T03:42:39,869 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e4336a99e9e54c15a4b9ebf5c1d21ac8_d3a8571217c3f763e5d9d9614d29758b store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:39,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742418_1594 (size=13017) 2024-11-11T03:42:39,873 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e4336a99e9e54c15a4b9ebf5c1d21ac8_d3a8571217c3f763e5d9d9614d29758b, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:39,873 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e4336a99e9e54c15a4b9ebf5c1d21ac8_d3a8571217c3f763e5d9d9614d29758b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:39,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742419_1595 (size=4469) 2024-11-11T03:42:40,275 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/932cb6b9bd394925aab40422781202c8 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/932cb6b9bd394925aab40422781202c8 2024-11-11T03:42:40,277 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:40,278 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/fab43c3f5f7f4459bff44ebeb783d5db is 175, key is test_row_0/A:col10/1731296558153/Put/seqid=0 2024-11-11T03:42:40,279 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/B of d3a8571217c3f763e5d9d9614d29758b into 932cb6b9bd394925aab40422781202c8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:40,280 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:40,280 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/B, priority=13, startTime=1731296559857; duration=0sec 2024-11-11T03:42:40,280 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T03:42:40,280 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:B 2024-11-11T03:42:40,280 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T03:42:40,281 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T03:42:40,281 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1541): d3a8571217c3f763e5d9d9614d29758b/C is initiating minor compaction (all files) 2024-11-11T03:42:40,281 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d3a8571217c3f763e5d9d9614d29758b/C in TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:40,281 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/6030fa6c5d1e40baba6069a96382f387, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/028705d5904c43959926b50416b4a175, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/57414df99f5943c9a4c0aac569321046] into tmpdir=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp, totalSize=36.6 K 2024-11-11T03:42:40,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742420_1596 (size=31971) 2024-11-11T03:42:40,282 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 6030fa6c5d1e40baba6069a96382f387, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731296553604 2024-11-11T03:42:40,282 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 028705d5904c43959926b50416b4a175, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731296555811 2024-11-11T03:42:40,282 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] compactions.Compactor(225): Compacting 57414df99f5943c9a4c0aac569321046, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731296557008 2024-11-11T03:42:40,288 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d3a8571217c3f763e5d9d9614d29758b#C#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T03:42:40,289 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/e2d63a5142fd4fe69c41569e7789c195 is 50, key is test_row_0/C:col10/1731296558153/Put/seqid=0 2024-11-11T03:42:40,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742421_1597 (size=13017) 2024-11-11T03:42:40,693 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/fab43c3f5f7f4459bff44ebeb783d5db as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fab43c3f5f7f4459bff44ebeb783d5db 2024-11-11T03:42:40,699 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/e2d63a5142fd4fe69c41569e7789c195 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e2d63a5142fd4fe69c41569e7789c195 2024-11-11T03:42:40,700 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/A of d3a8571217c3f763e5d9d9614d29758b into fab43c3f5f7f4459bff44ebeb783d5db(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:40,701 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:40,701 INFO [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/A, priority=13, startTime=1731296559857; duration=0sec 2024-11-11T03:42:40,701 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:40,701 DEBUG [RS:0;3ffe7f9d089e:34473-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:A 2024-11-11T03:42:40,706 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d3a8571217c3f763e5d9d9614d29758b/C of d3a8571217c3f763e5d9d9614d29758b into e2d63a5142fd4fe69c41569e7789c195(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T03:42:40,706 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d3a8571217c3f763e5d9d9614d29758b: 2024-11-11T03:42:40,706 INFO [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., storeName=d3a8571217c3f763e5d9d9614d29758b/C, priority=13, startTime=1731296559857; duration=0sec 2024-11-11T03:42:40,706 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T03:42:40,706 DEBUG [RS:0;3ffe7f9d089e:34473-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d3a8571217c3f763e5d9d9614d29758b:C 2024-11-11T03:42:42,622 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:42:46,256 INFO [Thread-2279 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2279. 2024-11-11T03:42:46,257 DEBUG [Thread-2279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:46,257 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,257 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:46,303 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-11T03:42:46,303 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:46,303 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,303 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:46,344 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-11T03:42:46,344 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T03:42:46,344 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,344 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3279 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3340 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3256 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3266 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3347 2024-11-11T03:42:46,345 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T03:42:46,346 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:42:46,346 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:42:46,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:46,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:46,347 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T03:42:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T03:42:46,352 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296566352"}]},"ts":"1731296566352"} 2024-11-11T03:42:46,354 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T03:42:46,354 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T03:42:46,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T03:42:46,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, UNASSIGN}] 2024-11-11T03:42:46,356 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, UNASSIGN 2024-11-11T03:42:46,356 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=CLOSING, regionLocation=3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:46,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, UNASSIGN because future has completed 2024-11-11T03:42:46,358 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T03:42:46,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288}] 2024-11-11T03:42:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T03:42:46,511 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:46,511 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T03:42:46,511 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing d3a8571217c3f763e5d9d9614d29758b, disabling compactions & flushes 2024-11-11T03:42:46,511 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:46,511 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:46,511 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. after waiting 0 ms 2024-11-11T03:42:46,512 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:46,512 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing d3a8571217c3f763e5d9d9614d29758b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-11T03:42:46,512 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=A 2024-11-11T03:42:46,512 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:46,513 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=B 2024-11-11T03:42:46,513 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:46,513 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d3a8571217c3f763e5d9d9614d29758b, store=C 2024-11-11T03:42:46,513 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T03:42:46,521 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115afdbd55a7ad4869aae26ee0d3dea90f_d3a8571217c3f763e5d9d9614d29758b is 50, key is test_row_0/A:col10/1731296566253/Put/seqid=0 2024-11-11T03:42:46,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742422_1598 (size=9914) 2024-11-11T03:42:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T03:42:46,928 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:42:46,938 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115afdbd55a7ad4869aae26ee0d3dea90f_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115afdbd55a7ad4869aae26ee0d3dea90f_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:46,940 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/719966f6c8bd4f4083069f84b4cf543c, store: [table=TestAcidGuarantees family=A region=d3a8571217c3f763e5d9d9614d29758b] 2024-11-11T03:42:46,941 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/719966f6c8bd4f4083069f84b4cf543c is 175, key is test_row_0/A:col10/1731296566253/Put/seqid=0 2024-11-11T03:42:46,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742423_1599 (size=22561) 2024-11-11T03:42:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T03:42:47,347 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/719966f6c8bd4f4083069f84b4cf543c 2024-11-11T03:42:47,392 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/961e58313d744d7093b7f8a963b985c0 is 50, key is test_row_0/B:col10/1731296566253/Put/seqid=0 2024-11-11T03:42:47,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742424_1600 (size=9857) 2024-11-11T03:42:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T03:42:47,797 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/961e58313d744d7093b7f8a963b985c0 2024-11-11T03:42:47,808 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0fe0ae037a374a1c97033299d7305ca2 is 50, key is test_row_0/C:col10/1731296566253/Put/seqid=0 2024-11-11T03:42:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742425_1601 (size=9857) 2024-11-11T03:42:48,214 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0fe0ae037a374a1c97033299d7305ca2 2024-11-11T03:42:48,223 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/A/719966f6c8bd4f4083069f84b4cf543c as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/719966f6c8bd4f4083069f84b4cf543c 2024-11-11T03:42:48,230 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/719966f6c8bd4f4083069f84b4cf543c, entries=100, sequenceid=308, filesize=22.0 K 2024-11-11T03:42:48,230 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/B/961e58313d744d7093b7f8a963b985c0 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/961e58313d744d7093b7f8a963b985c0 2024-11-11T03:42:48,234 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/961e58313d744d7093b7f8a963b985c0, entries=100, sequenceid=308, filesize=9.6 K 2024-11-11T03:42:48,234 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/.tmp/C/0fe0ae037a374a1c97033299d7305ca2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0fe0ae037a374a1c97033299d7305ca2 2024-11-11T03:42:48,238 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0fe0ae037a374a1c97033299d7305ca2, entries=100, sequenceid=308, filesize=9.6 K 2024-11-11T03:42:48,239 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d3a8571217c3f763e5d9d9614d29758b in 1727ms, sequenceid=308, compaction requested=false 2024-11-11T03:42:48,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/2721487713ed4739bd3999e633276301, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/41171f81afd94b8390a718d0fa19bc1a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/42971d894cf8472780eace9c5b6bd89a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/0818b9b105b74bce83bd76c95689144c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/d5cfe9ff454f450394ff6020b36bd4c9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/bbdc543a9c5747a7b2fa672348dfc096, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/e8c55d006f7d4a2980b9beb9c3a9ed59, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/35956794e04f49d297be9e1b8321676a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/4d8f000a49c945c18a10363bc7e83708, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/8deef19dab414aafb2b4a146f27341d7, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/c59a6060756547d194d3878481d58cf8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fffc9b47127b47308ba6b0f39ecd57ce, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/9423cbb989fb460ba9983436c202f555, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f1b54a4ae84349d1a20b0539c8a349ff, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f69af747396f4a1fb8bd398872b39e99, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/66a2b4bf1823476987e411bf42497e59, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/43dfb4c090004ec5923555326c576ead, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/09b2e6c1464c498c9dac2cc9455783b9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/b37eaee2b3a74837b8d654936625b500, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/81db6c4efd6f4a44ab91fb1437844ca1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/50c1c3dab2484d5aa28562b75f7a7b9a] to archive 2024-11-11T03:42:48,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:42:48,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/2721487713ed4739bd3999e633276301 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/2721487713ed4739bd3999e633276301 2024-11-11T03:42:48,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/41171f81afd94b8390a718d0fa19bc1a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/41171f81afd94b8390a718d0fa19bc1a 2024-11-11T03:42:48,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/42971d894cf8472780eace9c5b6bd89a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/42971d894cf8472780eace9c5b6bd89a 2024-11-11T03:42:48,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/0818b9b105b74bce83bd76c95689144c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/0818b9b105b74bce83bd76c95689144c 2024-11-11T03:42:48,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/d5cfe9ff454f450394ff6020b36bd4c9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/d5cfe9ff454f450394ff6020b36bd4c9 2024-11-11T03:42:48,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/bbdc543a9c5747a7b2fa672348dfc096 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/bbdc543a9c5747a7b2fa672348dfc096 2024-11-11T03:42:48,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/e8c55d006f7d4a2980b9beb9c3a9ed59 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/e8c55d006f7d4a2980b9beb9c3a9ed59 2024-11-11T03:42:48,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/35956794e04f49d297be9e1b8321676a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/35956794e04f49d297be9e1b8321676a 2024-11-11T03:42:48,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/4d8f000a49c945c18a10363bc7e83708 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/4d8f000a49c945c18a10363bc7e83708 2024-11-11T03:42:48,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/8deef19dab414aafb2b4a146f27341d7 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/8deef19dab414aafb2b4a146f27341d7 2024-11-11T03:42:48,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/c59a6060756547d194d3878481d58cf8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/c59a6060756547d194d3878481d58cf8 2024-11-11T03:42:48,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fffc9b47127b47308ba6b0f39ecd57ce to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fffc9b47127b47308ba6b0f39ecd57ce 2024-11-11T03:42:48,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/9423cbb989fb460ba9983436c202f555 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/9423cbb989fb460ba9983436c202f555 2024-11-11T03:42:48,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f1b54a4ae84349d1a20b0539c8a349ff to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f1b54a4ae84349d1a20b0539c8a349ff 2024-11-11T03:42:48,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f69af747396f4a1fb8bd398872b39e99 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/f69af747396f4a1fb8bd398872b39e99 2024-11-11T03:42:48,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/66a2b4bf1823476987e411bf42497e59 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/66a2b4bf1823476987e411bf42497e59 2024-11-11T03:42:48,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/43dfb4c090004ec5923555326c576ead to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/43dfb4c090004ec5923555326c576ead 2024-11-11T03:42:48,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/09b2e6c1464c498c9dac2cc9455783b9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/09b2e6c1464c498c9dac2cc9455783b9 2024-11-11T03:42:48,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/b37eaee2b3a74837b8d654936625b500 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/b37eaee2b3a74837b8d654936625b500 2024-11-11T03:42:48,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/81db6c4efd6f4a44ab91fb1437844ca1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/81db6c4efd6f4a44ab91fb1437844ca1 2024-11-11T03:42:48,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/50c1c3dab2484d5aa28562b75f7a7b9a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/50c1c3dab2484d5aa28562b75f7a7b9a 2024-11-11T03:42:48,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3ea30350754c4b7f8f079bbe463b1d54, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6b32a2e866094aad8a786fea59fc3150, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/74d2bf713d8a4615a733f86a5b75d16d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/1cb490baf51647eeaaacb0e9b14dee13, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/b8437efbb6c94d72882dd28bd9b1e5b0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6a81c306f0dd4db995adb8f30136bec1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e183766c0fe745fa9975eb6d466e74e8, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0937fa8a2e344a6da5fd48ac04300353, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/11d8c1f06dd0428f92840e3719bb82c0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/632328cf448f417582e4b64733a046e9, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3fec0de11e744e2988f2b89f8cce002c, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0e7d8f4335844e688ba004763aa638c5, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/078adc1b22c84a78a1af7cdc55ce68fd, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/5201a47c179d4aa5be73bff345cbef0d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/4a8b0865896646cab48d2e7374c4520d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/edd39481b40e4275aa26a5afa73d21ee, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/7eb1af0986ef4b288f810494459ae585, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e31d5e793a8542c598bb14678e05531f, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/58b8d26478824692b09ca57ffb028036, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/2e3edeb2bdfd4b78909894ccea3082a0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/54f8ac459c34481e8340e39b85eb3452] to archive 2024-11-11T03:42:48,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:42:48,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3ea30350754c4b7f8f079bbe463b1d54 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3ea30350754c4b7f8f079bbe463b1d54 2024-11-11T03:42:48,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6b32a2e866094aad8a786fea59fc3150 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6b32a2e866094aad8a786fea59fc3150 2024-11-11T03:42:48,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/74d2bf713d8a4615a733f86a5b75d16d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/74d2bf713d8a4615a733f86a5b75d16d 2024-11-11T03:42:48,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/1cb490baf51647eeaaacb0e9b14dee13 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/1cb490baf51647eeaaacb0e9b14dee13 2024-11-11T03:42:48,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/b8437efbb6c94d72882dd28bd9b1e5b0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/b8437efbb6c94d72882dd28bd9b1e5b0 2024-11-11T03:42:48,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6a81c306f0dd4db995adb8f30136bec1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/6a81c306f0dd4db995adb8f30136bec1 2024-11-11T03:42:48,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e183766c0fe745fa9975eb6d466e74e8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e183766c0fe745fa9975eb6d466e74e8 2024-11-11T03:42:48,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0937fa8a2e344a6da5fd48ac04300353 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0937fa8a2e344a6da5fd48ac04300353 2024-11-11T03:42:48,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/11d8c1f06dd0428f92840e3719bb82c0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/11d8c1f06dd0428f92840e3719bb82c0 2024-11-11T03:42:48,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/632328cf448f417582e4b64733a046e9 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/632328cf448f417582e4b64733a046e9 2024-11-11T03:42:48,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3fec0de11e744e2988f2b89f8cce002c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/3fec0de11e744e2988f2b89f8cce002c 2024-11-11T03:42:48,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0e7d8f4335844e688ba004763aa638c5 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/0e7d8f4335844e688ba004763aa638c5 2024-11-11T03:42:48,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/078adc1b22c84a78a1af7cdc55ce68fd to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/078adc1b22c84a78a1af7cdc55ce68fd 2024-11-11T03:42:48,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/5201a47c179d4aa5be73bff345cbef0d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/5201a47c179d4aa5be73bff345cbef0d 2024-11-11T03:42:48,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/4a8b0865896646cab48d2e7374c4520d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/4a8b0865896646cab48d2e7374c4520d 2024-11-11T03:42:48,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/edd39481b40e4275aa26a5afa73d21ee to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/edd39481b40e4275aa26a5afa73d21ee 2024-11-11T03:42:48,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/7eb1af0986ef4b288f810494459ae585 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/7eb1af0986ef4b288f810494459ae585 2024-11-11T03:42:48,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e31d5e793a8542c598bb14678e05531f to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/e31d5e793a8542c598bb14678e05531f 2024-11-11T03:42:48,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/58b8d26478824692b09ca57ffb028036 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/58b8d26478824692b09ca57ffb028036 2024-11-11T03:42:48,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/2e3edeb2bdfd4b78909894ccea3082a0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/2e3edeb2bdfd4b78909894ccea3082a0 2024-11-11T03:42:48,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/54f8ac459c34481e8340e39b85eb3452 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/54f8ac459c34481e8340e39b85eb3452 2024-11-11T03:42:48,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/5bfeef99a6c7431ba560a606268dfb05, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/d28bf357413d4303a874332aea4dde9a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a5c8aa0a69a4505a2365a89af82f6d0, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b7f34dacffc04b13bfadb286f874cf34, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/3ca7c9ed11c74aebbb52c8fa1d940953, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ffeb90c21ac44132888c34aaf029ea7d, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ef77206a1b3e4b3c99100360d4ff6329, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f0434354192c42cc99cc5f35f16ab935, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f8cfb4671863495d9ff138461e02b012, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/8739f03e2450489fb10f257d1a9e2801, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b4088b5a24cd439d9114e55ed38931a1, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/db24ff949a01484db1561292d0d8babf, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a7fd657eac94e9aacdc40db472872de, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/a3f1c390e881491abdcc66ba19c8596e, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b0a2c1bdfafc463ebd70853a88551fa3, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/27fb5928f64c438cba7be5328972c31a, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/c9186ba3d6724700a4a92970e61d5489, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/6030fa6c5d1e40baba6069a96382f387, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e6320f2736ea4392a32996a169ebc288, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/028705d5904c43959926b50416b4a175, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/57414df99f5943c9a4c0aac569321046] to archive 2024-11-11T03:42:48,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T03:42:48,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/5bfeef99a6c7431ba560a606268dfb05 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/5bfeef99a6c7431ba560a606268dfb05 2024-11-11T03:42:48,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/d28bf357413d4303a874332aea4dde9a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/d28bf357413d4303a874332aea4dde9a 2024-11-11T03:42:48,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a5c8aa0a69a4505a2365a89af82f6d0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a5c8aa0a69a4505a2365a89af82f6d0 2024-11-11T03:42:48,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b7f34dacffc04b13bfadb286f874cf34 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b7f34dacffc04b13bfadb286f874cf34 2024-11-11T03:42:48,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/3ca7c9ed11c74aebbb52c8fa1d940953 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/3ca7c9ed11c74aebbb52c8fa1d940953 2024-11-11T03:42:48,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ffeb90c21ac44132888c34aaf029ea7d to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ffeb90c21ac44132888c34aaf029ea7d 2024-11-11T03:42:48,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ef77206a1b3e4b3c99100360d4ff6329 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/ef77206a1b3e4b3c99100360d4ff6329 2024-11-11T03:42:48,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f0434354192c42cc99cc5f35f16ab935 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f0434354192c42cc99cc5f35f16ab935 2024-11-11T03:42:48,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f8cfb4671863495d9ff138461e02b012 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/f8cfb4671863495d9ff138461e02b012 2024-11-11T03:42:48,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/8739f03e2450489fb10f257d1a9e2801 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/8739f03e2450489fb10f257d1a9e2801 2024-11-11T03:42:48,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b4088b5a24cd439d9114e55ed38931a1 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b4088b5a24cd439d9114e55ed38931a1 2024-11-11T03:42:48,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/db24ff949a01484db1561292d0d8babf to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/db24ff949a01484db1561292d0d8babf 2024-11-11T03:42:48,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a7fd657eac94e9aacdc40db472872de to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0a7fd657eac94e9aacdc40db472872de 2024-11-11T03:42:48,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/a3f1c390e881491abdcc66ba19c8596e to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/a3f1c390e881491abdcc66ba19c8596e 2024-11-11T03:42:48,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b0a2c1bdfafc463ebd70853a88551fa3 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/b0a2c1bdfafc463ebd70853a88551fa3 2024-11-11T03:42:48,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/27fb5928f64c438cba7be5328972c31a to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/27fb5928f64c438cba7be5328972c31a 2024-11-11T03:42:48,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/c9186ba3d6724700a4a92970e61d5489 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/c9186ba3d6724700a4a92970e61d5489 2024-11-11T03:42:48,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/6030fa6c5d1e40baba6069a96382f387 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/6030fa6c5d1e40baba6069a96382f387 2024-11-11T03:42:48,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e6320f2736ea4392a32996a169ebc288 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e6320f2736ea4392a32996a169ebc288 2024-11-11T03:42:48,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/028705d5904c43959926b50416b4a175 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/028705d5904c43959926b50416b4a175 2024-11-11T03:42:48,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/57414df99f5943c9a4c0aac569321046 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/57414df99f5943c9a4c0aac569321046 2024-11-11T03:42:48,290 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/recovered.edits/311.seqid, newMaxSeqId=311, maxSeqId=4 2024-11-11T03:42:48,291 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. 2024-11-11T03:42:48,291 DEBUG [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for d3a8571217c3f763e5d9d9614d29758b: Waiting for close lock at 1731296566511Running coprocessor pre-close hooks at 1731296566511Disabling compacts and flushes for region at 1731296566511Disabling writes for close at 1731296566511Obtaining lock to block concurrent updates at 1731296566512 (+1 ms)Preparing flush snapshotting stores in d3a8571217c3f763e5d9d9614d29758b at 1731296566512Finished memstore snapshotting TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731296566513 (+1 ms)Flushing stores of TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b. at 1731296566514 (+1 ms)Flushing d3a8571217c3f763e5d9d9614d29758b/A: creating writer at 1731296566514Flushing mob file d3a8571217c3f763e5d9d9614d29758b/A: appending metadata at 1731296566521 (+7 ms)Flushing mob file d3a8571217c3f763e5d9d9614d29758b/A: closing flushed file at 1731296566521Flushing d3a8571217c3f763e5d9d9614d29758b/A: appending metadata at 1731296566940 (+419 ms)Flushing d3a8571217c3f763e5d9d9614d29758b/A: closing flushed file at 1731296566940Flushing d3a8571217c3f763e5d9d9614d29758b/B: creating writer at 1731296567388 (+448 ms)Flushing d3a8571217c3f763e5d9d9614d29758b/B: appending metadata at 1731296567391 (+3 ms)Flushing d3a8571217c3f763e5d9d9614d29758b/B: closing flushed file at 1731296567392 (+1 ms)Flushing d3a8571217c3f763e5d9d9614d29758b/C: creating writer at 1731296567803 (+411 ms)Flushing d3a8571217c3f763e5d9d9614d29758b/C: appending metadata at 1731296567807 (+4 ms)Flushing d3a8571217c3f763e5d9d9614d29758b/C: closing flushed file at 1731296567807Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25adf815: reopening flushed file at 1731296568222 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23431b84: reopening flushed file at 1731296568230 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78b1a6c7: reopening flushed file at 1731296568234 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d3a8571217c3f763e5d9d9614d29758b in 1727ms, sequenceid=308, compaction requested=false at 1731296568239 (+5 ms)Writing region close event to WAL at 1731296568289 (+50 ms)Running coprocessor post-close hooks at 1731296568291 (+2 ms)Closed at 1731296568291 2024-11-11T03:42:48,292 INFO [RS_CLOSE_REGION-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,293 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=d3a8571217c3f763e5d9d9614d29758b, regionState=CLOSED 2024-11-11T03:42:48,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 because future has completed 2024-11-11T03:42:48,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-11T03:42:48,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure d3a8571217c3f763e5d9d9614d29758b, server=3ffe7f9d089e,34473,1731296384288 in 1.9360 sec 2024-11-11T03:42:48,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-11T03:42:48,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d3a8571217c3f763e5d9d9614d29758b, UNASSIGN in 1.9400 sec 2024-11-11T03:42:48,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-11T03:42:48,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9430 sec 2024-11-11T03:42:48,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731296568299"}]},"ts":"1731296568299"} 2024-11-11T03:42:48,300 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T03:42:48,300 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T03:42:48,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9520 sec 2024-11-11T03:42:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T03:42:48,502 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:48,503 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T03:42:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:48,504 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T03:42:48,505 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:48,507 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,509 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C, FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/recovered.edits] 2024-11-11T03:42:48,512 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/719966f6c8bd4f4083069f84b4cf543c to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/719966f6c8bd4f4083069f84b4cf543c 2024-11-11T03:42:48,513 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fab43c3f5f7f4459bff44ebeb783d5db to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/A/fab43c3f5f7f4459bff44ebeb783d5db 2024-11-11T03:42:48,516 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/932cb6b9bd394925aab40422781202c8 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/932cb6b9bd394925aab40422781202c8 2024-11-11T03:42:48,517 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/961e58313d744d7093b7f8a963b985c0 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/B/961e58313d744d7093b7f8a963b985c0 2024-11-11T03:42:48,519 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0fe0ae037a374a1c97033299d7305ca2 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/0fe0ae037a374a1c97033299d7305ca2 2024-11-11T03:42:48,521 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e2d63a5142fd4fe69c41569e7789c195 to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/C/e2d63a5142fd4fe69c41569e7789c195 2024-11-11T03:42:48,524 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/recovered.edits/311.seqid to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b/recovered.edits/311.seqid 2024-11-11T03:42:48,525 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/default/TestAcidGuarantees/d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,525 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T03:42:48,525 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T03:42:48,526 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T03:42:48,530 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124a58db8c26e4895b4f06c0c0a8b4a69_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124a58db8c26e4895b4f06c0c0a8b4a69_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,531 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111126d5ebbd85a449df99224c0a4799ffbb_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111126d5ebbd85a449df99224c0a4799ffbb_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,533 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115afdbd55a7ad4869aae26ee0d3dea90f_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115afdbd55a7ad4869aae26ee0d3dea90f_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,535 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164ea573d9851475a9c74c184d9e6d571_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164ea573d9851475a9c74c184d9e6d571_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,536 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111740587164e0d4bb5a6b57de27e2923f4_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111740587164e0d4bb5a6b57de27e2923f4_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,538 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111177d6a0a0de5a43859348992a3878d3f7_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111177d6a0a0de5a43859348992a3878d3f7_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,540 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111182dc8ecc5a0b4054abbf13c2efba56be_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111182dc8ecc5a0b4054abbf13c2efba56be_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,541 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111184ac84f725e34349b7fb6d8acfd8a1b6_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111184ac84f725e34349b7fb6d8acfd8a1b6_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,543 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111189f93b77a966405288b91e354a408609_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111189f93b77a966405288b91e354a408609_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,544 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111190400ac047de489aac0afadc671ac084_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111190400ac047de489aac0afadc671ac084_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,545 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195b2554c2c2c4f3fb2f9f92d697eeb2e_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195b2554c2c2c4f3fb2f9f92d697eeb2e_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,547 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b1158207b18c4a8eaa912c64edc90b67_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b1158207b18c4a8eaa912c64edc90b67_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,548 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf805a018d5c4591a6caa2565a371d2b_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf805a018d5c4591a6caa2565a371d2b_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,549 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0bc275ab95247308b269a9dacfcdf0d_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0bc275ab95247308b269a9dacfcdf0d_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,550 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e753f05d49b94c33944da824f067942d_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e753f05d49b94c33944da824f067942d_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,552 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fe9e5fe3604745daa7c862925cbd7270_d3a8571217c3f763e5d9d9614d29758b to hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fe9e5fe3604745daa7c862925cbd7270_d3a8571217c3f763e5d9d9614d29758b 2024-11-11T03:42:48,552 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T03:42:48,555 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:48,559 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T03:42:48,562 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T03:42:48,563 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:48,563 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T03:42:48,563 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731296568563"}]},"ts":"9223372036854775807"} 2024-11-11T03:42:48,566 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T03:42:48,566 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d3a8571217c3f763e5d9d9614d29758b, NAME => 'TestAcidGuarantees,,1731296534079.d3a8571217c3f763e5d9d9614d29758b.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T03:42:48,566 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T03:42:48,566 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731296568566"}]},"ts":"9223372036854775807"} 2024-11-11T03:42:48,567 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T03:42:48,568 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T03:42:48,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-11T03:42:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T03:42:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T03:42:48,613 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T03:42:48,626 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 312), ProcessCount=11 (was 11), AvailableMemoryMB=11776 (was 11797) 2024-11-11T03:42:48,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T03:42:48,626 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:42:48,626 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:42:48,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:48,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:48,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T03:42:48,627 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T03:42:48,627 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-11-11T03:42:48,627 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3ffe7f9d089e,45295,1731296383499 2024-11-11T03:42:48,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T03:42:48,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T03:42:48,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:42:48,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:42:48,684 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T03:42:48,685 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:42:48,686 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T03:42:48,686 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:42:48,686 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:42:48,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:48,687 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3ffe7f9d089e,34473,1731296384288' ***** 2024-11-11T03:42:48,687 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T03:42:48,688 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T03:42:48,688 INFO [RS:0;3ffe7f9d089e:34473 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T03:42:48,689 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(959): stopping server 3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3ffe7f9d089e:34473. 2024-11-11T03:42:48,689 DEBUG [RS:0;3ffe7f9d089e:34473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:42:48,689 DEBUG [RS:0;3ffe7f9d089e:34473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T03:42:48,689 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T03:42:48,690 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T03:42:48,690 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T03:42:48,690 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:48,690 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T03:42:48,690 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T03:42:48,690 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T03:42:48,690 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T03:42:48,690 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T03:42:48,690 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T03:42:48,712 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/info/fd403e8c8bf848a39782864b67fe16f5 is 95, key is TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358./info:/1731296418830/DeleteFamily/seqid=0 2024-11-11T03:42:48,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742426_1602 (size=6514) 2024-11-11T03:42:48,771 INFO [regionserver/3ffe7f9d089e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T03:42:48,890 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:49,091 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:49,116 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/info/fd403e8c8bf848a39782864b67fe16f5 2024-11-11T03:42:49,141 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/ns/a5b492dbce8646cba486c8bddc10c84f is 93, key is TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358./ns:/1731296418793/DeleteFamily/seqid=0 2024-11-11T03:42:49,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742427_1603 (size=5906) 2024-11-11T03:42:49,285 INFO [regionserver/3ffe7f9d089e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T03:42:49,285 INFO [regionserver/3ffe7f9d089e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T03:42:49,291 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:49,491 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:49,545 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/ns/a5b492dbce8646cba486c8bddc10c84f 2024-11-11T03:42:49,571 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/rep_barrier/ab9c9bfc3c514c10a59e5e6529a48de6 is 102, key is TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358./rep_barrier:/1731296418793/DeleteFamily/seqid=0 2024-11-11T03:42:49,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742428_1604 (size=6025) 2024-11-11T03:42:49,692 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:49,892 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:49,976 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/rep_barrier/ab9c9bfc3c514c10a59e5e6529a48de6 2024-11-11T03:42:50,005 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/table/78e6695c0343414baece91196e87fc19 is 96, key is TestAcidGuarantees,,1731296387799.488bb5013d196b74e2d5128b2d5b7358./table:/1731296418793/DeleteFamily/seqid=0 2024-11-11T03:42:50,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742429_1605 (size=5936) 2024-11-11T03:42:50,092 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:50,292 DEBUG [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T03:42:50,409 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/table/78e6695c0343414baece91196e87fc19 2024-11-11T03:42:50,415 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/info/fd403e8c8bf848a39782864b67fe16f5 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/info/fd403e8c8bf848a39782864b67fe16f5 2024-11-11T03:42:50,421 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/info/fd403e8c8bf848a39782864b67fe16f5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T03:42:50,423 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/ns/a5b492dbce8646cba486c8bddc10c84f as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/ns/a5b492dbce8646cba486c8bddc10c84f 2024-11-11T03:42:50,427 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/ns/a5b492dbce8646cba486c8bddc10c84f, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T03:42:50,427 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/rep_barrier/ab9c9bfc3c514c10a59e5e6529a48de6 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/rep_barrier/ab9c9bfc3c514c10a59e5e6529a48de6 2024-11-11T03:42:50,430 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/rep_barrier/ab9c9bfc3c514c10a59e5e6529a48de6, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T03:42:50,431 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/.tmp/table/78e6695c0343414baece91196e87fc19 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/table/78e6695c0343414baece91196e87fc19 2024-11-11T03:42:50,434 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/table/78e6695c0343414baece91196e87fc19, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T03:42:50,435 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1745ms, sequenceid=90, compaction requested=false 2024-11-11T03:42:50,438 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T03:42:50,439 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T03:42:50,439 INFO [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T03:42:50,439 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731296568690Running coprocessor pre-close hooks at 1731296568690Disabling compacts and flushes for region at 1731296568690Disabling writes for close at 1731296568690Obtaining lock to block concurrent updates at 1731296568690Preparing flush snapshotting stores in 1588230740 at 1731296568690Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731296568691 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731296568692 (+1 ms)Flushing 1588230740/info: creating writer at 1731296568692Flushing 1588230740/info: appending metadata at 1731296568712 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731296568712Flushing 1588230740/ns: creating writer at 1731296569126 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731296569141 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731296569141Flushing 1588230740/rep_barrier: creating writer at 1731296569553 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731296569571 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731296569571Flushing 1588230740/table: creating writer at 1731296569984 (+413 ms)Flushing 1588230740/table: appending metadata at 1731296570005 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731296570005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13a67dbd: reopening flushed file at 1731296570414 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69ead440: reopening flushed file at 1731296570422 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71f83b31: reopening flushed file at 1731296570427 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cccb761: reopening flushed file at 1731296570430 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1745ms, sequenceid=90, compaction requested=false at 1731296570435 (+5 ms)Writing region close event to WAL at 1731296570436 (+1 ms)Running coprocessor post-close hooks at 1731296570439 (+3 ms)Closed at 1731296570439 2024-11-11T03:42:50,439 DEBUG [RS_CLOSE_META-regionserver/3ffe7f9d089e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T03:42:50,493 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(976): stopping server 3ffe7f9d089e,34473,1731296384288; all regions closed. 2024-11-11T03:42:50,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741834_1010 (size=25344) 2024-11-11T03:42:50,505 DEBUG [RS:0;3ffe7f9d089e:34473 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/oldWALs 2024-11-11T03:42:50,505 INFO [RS:0;3ffe7f9d089e:34473 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3ffe7f9d089e%2C34473%2C1731296384288.meta:.meta(num 1731296387345) 2024-11-11T03:42:50,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741833_1009 (size=14609149) 2024-11-11T03:42:50,509 DEBUG [RS:0;3ffe7f9d089e:34473 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/oldWALs 2024-11-11T03:42:50,509 INFO [RS:0;3ffe7f9d089e:34473 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3ffe7f9d089e%2C34473%2C1731296384288:(num 1731296386448) 2024-11-11T03:42:50,509 DEBUG [RS:0;3ffe7f9d089e:34473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:42:50,509 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T03:42:50,509 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T03:42:50,509 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.ChoreService(370): Chore service for: regionserver/3ffe7f9d089e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T03:42:50,510 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T03:42:50,510 INFO [regionserver/3ffe7f9d089e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T03:42:50,511 INFO [RS:0;3ffe7f9d089e:34473 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34473 2024-11-11T03:42:50,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3ffe7f9d089e,34473,1731296384288 2024-11-11T03:42:50,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T03:42:50,583 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T03:42:50,597 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3ffe7f9d089e,34473,1731296384288] 2024-11-11T03:42:50,608 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3ffe7f9d089e,34473,1731296384288 already deleted, retry=false 2024-11-11T03:42:50,608 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3ffe7f9d089e,34473,1731296384288 expired; onlineServers=0 2024-11-11T03:42:50,608 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3ffe7f9d089e,45295,1731296383499' ***** 2024-11-11T03:42:50,608 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T03:42:50,608 INFO [M:0;3ffe7f9d089e:45295 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T03:42:50,608 INFO [M:0;3ffe7f9d089e:45295 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T03:42:50,609 DEBUG [M:0;3ffe7f9d089e:45295 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T03:42:50,609 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T03:42:50,609 DEBUG [M:0;3ffe7f9d089e:45295 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T03:42:50,609 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster-HFileCleaner.large.0-1731296386029 {}] cleaner.HFileCleaner(306): Exit Thread[master/3ffe7f9d089e:0:becomeActiveMaster-HFileCleaner.large.0-1731296386029,5,FailOnTimeoutGroup] 2024-11-11T03:42:50,609 DEBUG [master/3ffe7f9d089e:0:becomeActiveMaster-HFileCleaner.small.0-1731296386031 {}] cleaner.HFileCleaner(306): Exit Thread[master/3ffe7f9d089e:0:becomeActiveMaster-HFileCleaner.small.0-1731296386031,5,FailOnTimeoutGroup] 2024-11-11T03:42:50,609 INFO [M:0;3ffe7f9d089e:45295 {}] hbase.ChoreService(370): Chore service for: master/3ffe7f9d089e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T03:42:50,609 INFO [M:0;3ffe7f9d089e:45295 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T03:42:50,609 DEBUG [M:0;3ffe7f9d089e:45295 {}] master.HMaster(1795): Stopping service threads 2024-11-11T03:42:50,609 INFO [M:0;3ffe7f9d089e:45295 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T03:42:50,610 INFO [M:0;3ffe7f9d089e:45295 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T03:42:50,610 ERROR [M:0;3ffe7f9d089e:45295 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T03:42:50,611 INFO [M:0;3ffe7f9d089e:45295 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T03:42:50,611 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T03:42:50,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T03:42:50,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:42:50,621 DEBUG [M:0;3ffe7f9d089e:45295 {}] zookeeper.ZKUtil(347): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T03:42:50,622 WARN [M:0;3ffe7f9d089e:45295 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T03:42:50,622 INFO [M:0;3ffe7f9d089e:45295 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/.lastflushedseqids 2024-11-11T03:42:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742430_1606 (size=72) 2024-11-11T03:42:50,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:42:50,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34473-0x1012943e1960001, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:42:50,699 INFO [RS:0;3ffe7f9d089e:34473 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T03:42:50,699 INFO [RS:0;3ffe7f9d089e:34473 {}] regionserver.HRegionServer(1031): Exiting; stopping=3ffe7f9d089e,34473,1731296384288; zookeeper connection closed. 2024-11-11T03:42:50,700 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f95a440 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f95a440 2024-11-11T03:42:50,701 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T03:42:51,039 INFO [M:0;3ffe7f9d089e:45295 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T03:42:51,040 INFO [M:0;3ffe7f9d089e:45295 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T03:42:51,040 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T03:42:51,040 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:42:51,040 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:42:51,040 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T03:42:51,040 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:42:51,041 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=778.07 KB heapSize=953.31 KB 2024-11-11T03:42:51,064 DEBUG [M:0;3ffe7f9d089e:45295 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21a6a452768545cc888404d5a5ccfcdd is 82, key is hbase:meta,,1/info:regioninfo/1731296387422/Put/seqid=0 2024-11-11T03:42:51,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742431_1607 (size=5672) 2024-11-11T03:42:51,468 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21a6a452768545cc888404d5a5ccfcdd 2024-11-11T03:42:51,494 DEBUG [M:0;3ffe7f9d089e:45295 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/741e2e61fc7842cb94b8b04783a3b983 is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1731296422034/Put/seqid=0 2024-11-11T03:42:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742432_1608 (size=45432) 2024-11-11T03:42:51,899 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=777.52 KB at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/741e2e61fc7842cb94b8b04783a3b983 2024-11-11T03:42:51,908 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 741e2e61fc7842cb94b8b04783a3b983 2024-11-11T03:42:51,922 DEBUG [M:0;3ffe7f9d089e:45295 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9130bee4f3c049919cdf78231c94bdf2 is 69, key is 3ffe7f9d089e,34473,1731296384288/rs:state/1731296386148/Put/seqid=0 2024-11-11T03:42:51,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073742433_1609 (size=5156) 2024-11-11T03:42:52,327 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2166 (bloomFilter=true), to=hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9130bee4f3c049919cdf78231c94bdf2 2024-11-11T03:42:52,334 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21a6a452768545cc888404d5a5ccfcdd as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/21a6a452768545cc888404d5a5ccfcdd 2024-11-11T03:42:52,340 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/21a6a452768545cc888404d5a5ccfcdd, entries=8, sequenceid=2166, filesize=5.5 K 2024-11-11T03:42:52,341 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/741e2e61fc7842cb94b8b04783a3b983 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/741e2e61fc7842cb94b8b04783a3b983 2024-11-11T03:42:52,344 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 741e2e61fc7842cb94b8b04783a3b983 2024-11-11T03:42:52,344 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/741e2e61fc7842cb94b8b04783a3b983, entries=170, sequenceid=2166, filesize=44.4 K 2024-11-11T03:42:52,345 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9130bee4f3c049919cdf78231c94bdf2 as hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9130bee4f3c049919cdf78231c94bdf2 2024-11-11T03:42:52,348 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44167/user/jenkins/test-data/bad9fe83-a7e7-530b-f024-1822a10cf78c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9130bee4f3c049919cdf78231c94bdf2, entries=1, sequenceid=2166, filesize=5.0 K 2024-11-11T03:42:52,349 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(3140): Finished flush of dataSize ~778.07 KB/796745, heapSize ~953.02 KB/975888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2166, compaction requested=false 2024-11-11T03:42:52,350 INFO [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:42:52,350 DEBUG [M:0;3ffe7f9d089e:45295 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731296571040Disabling compacts and flushes for region at 1731296571040Disabling writes for close at 1731296571040Obtaining lock to block concurrent updates at 1731296571041 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731296571041Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=796745, getHeapSize=976128, getOffHeapSize=0, getCellsCount=2361 at 1731296571041Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731296571043 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731296571043Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731296571063 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731296571063Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731296571476 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731296571494 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731296571494Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731296571908 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731296571922 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731296571922Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4200ba80: reopening flushed file at 1731296572333 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@662ec6cd: reopening flushed file at 1731296572340 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75073422: reopening flushed file at 1731296572344 (+4 ms)Finished flush of dataSize ~778.07 KB/796745, heapSize ~953.02 KB/975888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2166, compaction requested=false at 1731296572349 (+5 ms)Writing region close event to WAL at 1731296572350 (+1 ms)Closed at 1731296572350 2024-11-11T03:42:52,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741830_1006 (size=938734) 2024-11-11T03:42:52,397 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T03:42:52,397 INFO [M:0;3ffe7f9d089e:45295 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T03:42:52,398 INFO [M:0;3ffe7f9d089e:45295 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45295 2024-11-11T03:42:52,398 INFO [M:0;3ffe7f9d089e:45295 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T03:42:52,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:42:52,541 INFO [M:0;3ffe7f9d089e:45295 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T03:42:52,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45295-0x1012943e1960000, quorum=127.0.0.1:50016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:42:52,553 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T03:42:52,555 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T03:42:52,555 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T03:42:52,556 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T03:42:52,556 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/hadoop.log.dir/,STOPPED} 2024-11-11T03:42:52,558 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T03:42:52,558 WARN [BP-220414605-172.17.0.2-1731296379415 heartbeating to localhost/127.0.0.1:44167 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T03:42:52,558 WARN [BP-220414605-172.17.0.2-1731296379415 heartbeating to localhost/127.0.0.1:44167 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-220414605-172.17.0.2-1731296379415 (Datanode Uuid 0c3298f7-bd01-4a84-915f-4c7cc85b3801) service to localhost/127.0.0.1:44167 2024-11-11T03:42:52,558 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T03:42:52,561 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/data/data1/current/BP-220414605-172.17.0.2-1731296379415 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T03:42:52,561 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/cluster_83e25289-8713-26a3-4e13-a4ea80f515f5/data/data2/current/BP-220414605-172.17.0.2-1731296379415 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T03:42:52,562 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T03:42:52,569 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T03:42:52,570 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T03:42:52,570 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T03:42:52,571 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T03:42:52,571 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d602c43e-1ccb-34d1-9b47-f5f2f918a85d/hadoop.log.dir/,STOPPED} 2024-11-11T03:42:52,584 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T03:42:52,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down